[ldmMcidas #JRE-535181]: NLDN LDM data decoder

Hi Ken,

re:
> I'm a researcher at The University of Oklahoma who is interested in
> looking at the NLDN data we are currently receiving.  I believe McIDAS
> and GEMPAK have built in decoders, but I need to see the actual data
> values.

Yes, the ldm-mcidas package contains a decoder that converts the NLDN data
sent in the IDD into McIDAS-compatible (MD) files, and Unidata GEMPAK
contains a decoder that converts the same data into GEMPAK-compatible files.

> I found some notes on the internet stating there is currently
> no decoder to convert to NetCDF format.  Is that correct?

There is no program available through Unidata that does this decoding.  I do
not know if someone in the community has created such a routine, however.

> I am currently
> working on a converter for the binary LDM access, but I have been unable
> to correctly decipher the data.  I've been following the documentation
> located here <http://www.unidata.ucar.edu/data/lightning.html> but the
> data files I have do not appear to match this format.

Hmm...  I updated the contents of the web page you reference to give specific
information on the data being sent in the LDM/IDD 'NLDN' datastream.  I am
surprised by your comment that the information does not correspond to what
you have, but I guess that it has mostly to do with the header found at
the beginning of each IDD product.

> Can you provide some more information or answer a few questions.
> 
> * How many bytes is the file header.  I make it out to be 84 bytes.
> * How long is each data record?  I make them out to be 28 bytes by
>   looking at patterns in the binary file, but would assume 33 from
>   the description in the table.
> * Is "NLDN" written at the start of each flash? (ASCII?)
> * What does the Size notation indicate in the Description table?
>   I'm assuming char[4] means 4 single character bytes and int[4]
>   means 1 integer of size 4 bytes.

I think that the routine I use in the ldm-mcidas package would answer
all of your questions, so I have attached it to this reply.

> * What is the fill/padding value used?

I am not sure.  We skip over the padding value as they contain no useful
information.

> * Are all the bytes signed or unsigned?

We treat them as unsigned.

> Any information you can provide me would be very helpful.

Please let me know if the source code (nldninput.c) answers your questions.

Cheers,

Tom
****************************************************************************
Unidata User Support                                    UCAR Unidata Program
(303) 497-8642                                                 P.O. Box 3000
address@hidden                                   Boulder, CO 80307
----------------------------------------------------------------------------
Unidata HomePage                       http://www.unidata.ucar.edu
****************************************************************************


Ticket Details
===================
Ticket ID: JRE-535181
Department: Support ldm-mcidas
Priority: Normal
Status: Closed

/* ---------------------------------------------------------------------
**
**  Name:     ingestnldn.c
**
**  Purpose:  Read NLDN binary records from LDM feed
**
**  History:  19940302 - Adapted from 'ingesttonc.c' written by Ron
**                       Henderson of SUNY Albany
**
** ---------------------------------------------------------------------
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <assert.h>
#include <signal.h>
#include "ulog.h"
#include "cfortran.h"
#include "alarm.h"

#define INGESTORFLASHLEN 28

static unsigned alarm_timeout = 60;     /* seconds; should suffice */

/*
 * SIGALRM handler.
 */
static void
alarm_handler( sig )
int           sig;
{
    assert( SIGALRM == sig );
    uerror( "nldninput(): no data read in %u seconds: simulating EOF", 
alarm_timeout );
}

/*
 * Read a character from a file.
 *
 * Returns:
 *      1       Success
 *      0       Failure
 */
static int
read_char( file, value )
FILE         *file;
char         *value;
{
    return fread(value, 1, 1, file) == 1;
}


/*
 * Read a short from a file.
 *
 * Returns:
 *       1      Success
 *       0      Failure
 */
static int
read_short( file, value )
FILE         *file;
short        *value;
{
    unsigned char x[2];                 /* NB: 16-bits assumed! */
    int         retcode;
    int         num = 1;

    if (fread(x, 1, sizeof(x), file) == sizeof(x))
    {
        *value = (x[0] << 8) | x[1];
        retcode = 1;
    }
    else
    {
        retcode = 0;
    }

    return retcode;
}


/*
 * Read an integer from a file.
 *
 * Returns:
 *       1      Success
 *       0      Failure
 */
static int
read_int( file, value )
FILE         *file;
int          *value;
{
    unsigned char x[4];                 /* NB: 32-bits assumed! */
    int         retcode;
    int         num = 1;

    if (fread(x, 1, sizeof(x), file) == sizeof(x))
    {
        *value = (x[0] << 24) | (x[1] << 16) | (x[2] << 8) | x[3];
        retcode = 1;
    }
    else
    {
        retcode = 0;
    }

    return retcode;
}

/*
 * Read an undetermined 4-byte sequence from a file; determine whether
 * or not it is a beginning of a header record; eat all header records
 * if they exist; and return the integer value at the beginning of a 
 * flash record.
 *
 * Returns:
 *       1      Success
 *       0      Failure
 */
static int
read_begin( file, value )
FILE         *file;
int          *value;
{
    unsigned char x[4];                 /* NB: 32-bits assumed! */
    int           retcode = 0;          /* failure default */
    int           num = 1;

    if (fread(x, 1, sizeof(x), file) == sizeof(x)) 
    {
        if (strncmp((const char *)x, "NLDN", 4) != 0)
        {
            /*
             * Not a header record.  Just return the data value.
             */
            retcode = 1;
        }
        else
        {
            /*
             * Start of header records.  Read them and the first data
             * value.
             */
            int           nhead;

            uinfo("nldninput(): Product header record found");
            if (read_int(file, &nhead))
            {
                int           nflash;

                if (read_int(file, &nflash))
                {
                    unsigned char headr[280];   /* NB: 32-bits assumed! */
                    int           bufsiz = nhead * INGESTORFLASHLEN - 12;

                    if (fread(&headr[12], 1, bufsiz, file) == bufsiz)
                    {
                        if (nflash != 0)
                        {
                            if (fread(x, 1, sizeof(x), file) == sizeof(x))
                                retcode = 1;
                        }
                    }
                }
            }
        }

        if (retcode == 1)
            *value = (x[0] << 24) | (x[1] << 16) | (x[2] << 8) | x[3];
    }

    return retcode;
}

/*
 * Read an NLDN record from a file stream.
 *
 * Returns:
 *      INGESTORFLASHLEN -> Success
 *      0                -> Failure
 */
static int
nldn_read( file, time_sec, time_nsec, lat, lon, sgnl, mult, semimaj, eccent,
           angle, chisqr )
FILE         *file;
int          *time_sec;
int          *time_nsec;
float        *lat;
float        *lon;
float        *sgnl;
int          *mult;
float        *semimaj;
float        *eccent;
float        *angle;
float        *chisqr;
{
    int   iret;
    int   nbytes;
    short fill_short;
    char  fill_char;
    int   rc = 0;                                               /* failure */
    struct {
      int   lat;
      int   lon;
      short sgnl;
      char  mult;
      char  semimaj;
      char  eccent;
      char  angle;
      char  chisqr;
    } obs;

    if ( read_begin(file, time_sec)    &&  /* 32-bits assumed */
         read_int(file, time_nsec)     &&  /* 32-bits assumed */
         read_int(file, &obs.lat)      &&
         read_int(file, &obs.lon)      &&
         read_short(file, &fill_short) &&
         read_short(file, &obs.sgnl)   &&
         read_short(file, &fill_short) &&
         read_char(file, &obs.mult)    &&
         read_char(file, &fill_char)   &&
         read_char(file, &obs.semimaj) &&
         read_char(file, &obs.eccent)  &&
         read_char(file, &obs.angle)   &&
         read_char(file, &obs.chisqr)     ) {

      *lat     = obs.lat / 1000.0;
      *lon     = obs.lon / 1000.0;
      *sgnl    = obs.sgnl / 10.0;
      *mult    = obs.mult;
      *semimaj = obs.semimaj;
      *eccent  = obs.eccent;
      *angle   = obs.angle;
      *chisqr  = obs.chisqr;

      rc = INGESTORFLASHLEN;

    }

    return( rc );

}


/*
 * Return parameters from the input.  In order to handle the situation in
 * which we're being fed by an LDM and the end of the data product is never
 * seen, we set an alarm prior to the read.
 *
 * Returns:
 *      -1                      Error: read error or EOF
 *       0                      No data
 *      INGESTORFLASHLEN        Number of bytes read
 */
int
nldninput(time_sec, time_nsec, lat, lon, sgnl, mult, semimaj, eccent,
          angle, chisqr)
int          *time_sec;
int          *time_nsec;
float        *lat;
float        *lon;
float        *sgnl;
int          *mult;
float        *semimaj;
float        *eccent;
float        *angle;
float        *chisqr;
{
    int           rc = -1;                              /* failure default */
    static int    alarm_initialized = 0;
    static Alarm  timeout_alarm;

    if ( !alarm_initialized ) {
          alarm_init( &timeout_alarm, alarm_timeout, alarm_handler );
          alarm_initialized = 1;
    }

    alarm_on(&timeout_alarm);                           /* set alarm */

    rc = nldn_read( stdin, time_sec, time_nsec, lat, lon, sgnl, mult,
                    semimaj, eccent, angle, chisqr );

    alarm_off( &timeout_alarm );                        /* disable alarm */

    if ( !rc ) {
          if ( feof(stdin) ) {
            uinfo( "nldninput(): EOF on stdin" );
          } else if ( ferror(stdin) ) {
            serror( "nldninput(): NLDN read error" );
          } else {
            uinfo( "nldninput(): No data to read" );
          }
    }

    return( rc );
}


/*
 * FORTRAN interface to the above function.
 */
FCALLSCFUN10(INT,nldninput,NLDNIN,nldnin,PINT,PINT,PFLOAT,PFLOAT,PFLOAT,PINT,PFLOAT,PFLOAT,PFLOAT,PFLOAT)