Re: 19990512: ingest of a teletype file into LDM?

Jeff,

The simplest solution would be to use new afos code that will create an
WMO type header with the AFOS feedtype. I know this is a kuldge but it
should work because the tty format is like AFOS.  The alternative would be
to use the new afos code as a template and make an EXP feedtype. This
probably would require  modifying pqing.c, creating tty.c and tty.h  and
more. 

Robb...


On Fri, 14 May 1999, Unidata Support wrote:

> 
> ------- Forwarded Message
> 
> >cc: address@hidden
> >From: Jeff Ator <address@hidden>
> >Subject: ingest of a teletype file into LDM?
> >Organization: National Centers for Environmental Prediction
> >Keywords: 199905121231.GAA08153 LDM teletype
> 
> Hello,
> 
> On our public ftp server is a copy of a file that we would like to be
> able to ingest into our LDM (version 5.0.5).  The file is from South
> Africa and appears to contain WMO bulletins in a teletype (tty) format,
> with a ZCZC beginning and NNNN end.  In peeking through the LDM source
> code, it appears that there is  the capability (via
> pqing/afos_message.c) to ingest AFOS pils that are in teletype format,
> but we couldn't find anything comparable for WMO bulletins.  Does such a
> utility exist and, if so, could you point me to it and/or offer other
> suggestions?  To help you investigate, a sample file is on our public
> ftp server.  FTP to 140.90.50.22 and login as anonymous (you'll be
> prompted for your email address as a password).  cd to pub/incomming,
> and the file is "messages_text" (be sure to use binary ftp!).
> 
> Thanks very much for any help you can provide!
> 
> Sincerely,
> __________________________________
> 
> Jeffrey B. Ator
> 
> National Centers for
>    Environmental Prediction
> Central Operations
> 5200 Auth Road, Rm. 307
> Camp Springs, MD  20746
> 
> Phone:  (301) 763-8000, Ext. 7104
> Fax:    (301) 763-8381
> 
> E-Mail:  address@hidden
> 
> __________________________________
> 
> 
> 
> ------- End of Forwarded Message
> 

===============================================================================
Robb Kambic                                Unidata Program Center
Software Engineer III                      Univ. Corp for Atmospheric Research
address@hidden             WWW: http://www.unidata.ucar.edu/
===============================================================================

/*
 *   Copyright 1995, University Corporation for Atmospheric Research
 *   See ../COPYRIGHT file for copying and redistribution conditions.
 */
/* $Id: afos_message.c,v 1.39 1999/04/20 22:25:42 davis Exp $ */

/* 
 * Routines to handle AFOS messages.
 * Thanks to NWS personnel, who shared their ldm3 mods and
 * other info.
 * N.B. This module prepared without any supporting definitions
 * or AFOS documentation.
 */

#include <ldmconfig.h>
#include <string.h>
#include <ctype.h>
#include "xbuf.h"
#include "tokens.h"
#include "afos_message.h"
#include "ulog.h"


static int seqno = 0;
static int zczc_missed = 0;
static int nnnn_missed = 0;
static int unrecognizable = 0;

void
afos_stats(void)
{
        unotice("  AFOS Messages seen: %8d", seqno - unrecognizable);
        unotice("  Errors:  lone ZCZC: %8d", nnnn_missed);
        unotice("           lone NNNN: %8d", zczc_missed);
        unotice("      Unrecognizable: %8d", unrecognizable);
}


/*
 * Set time stamp and hook up body of message
 */
static afos_message *
init_afos_message(
        xbuf *buf, /* should be a clone ! */
        afos_message *mess
)
{
        mess->len = buf->cnt;
        mess->msg = buf->get;
        
        if( set_timestamp(&mess->arrival) != 0)
                return NULL;

        return mess;
}


/*
 * skip up to the first Upper case char and get the rest of the line
 */
static int
nextUcaseLine(xbuf *buf, char *str, int maxlen) 
{
        int ch;
        do{
                ch = nextc(buf);
                if(ch == EOB)
                        break;
        }while(!isascii(ch) || !isupper(ch));
        unnextc(buf,ch);
        return( get_line(buf, str, maxlen) );
}

#define MIN_AFOS_MSG_LEN 14 /* wild ass guess */

static int
get_afos_message(xbuf *buf, afos_message *mess)
{
        int zczc_missing = 0;

        init_afos_message(buf, mess);

        /* DEBUG */
        if(mess->len < MIN_AFOS_MSG_LEN)
        {
                uerror("new_afos_message: length %d too short", mess->len);
                goto err;
        }

        if(!hasSTR(buf, "ZCZC"))
        {
                /* not fatal, complain later */
                zczc_missing = 1;
        }

        if(nextUcaseLine(buf, mess->ident, 12) == EOB) goto err;

        {
                /*
                 * Skip white space and (maybe) the next two bytes.
                 * What do they mean?
                 */
                int ch;
                do{
                        ch = nextc(buf);
                } while((isascii(ch) && !isgraph(ch)));
                if(isascii(ch))
                        unnextc(buf, ch);
                else
                        ch = nextc(buf);
        }
        
        {
                /* optional */
                wmo_header_t wmo_hdr;
                dtime time;
                wmo_hdr.time = &time;
                if(get_wmo_header(buf, &wmo_hdr) != NULL
                                && isupper(wmo_hdr.TT[0])
                                && isupper(wmo_hdr.TT[1])
                                && isupper(wmo_hdr.AA[0])
                                && isupper(wmo_hdr.CCCC[0])
                        )
                {
                        size_t len = strlen(mess->ident);
                        sprintf(&mess->ident[len],
                                 "   (%s)", s_wmo_header(&wmo_hdr));
                }
        }

        if(zczc_missing)
        {
                uerror(" Missing ZCZC: %6s %03d %8d  %s",
                                "AFOS",
                                seqno,
                                mess->len,
                                mess->ident);
        }
        if(memcmp(&mess->msg[mess->len -4], "NNNN", 4) != 0)
        {
                uerror(" Missing NNNN: %6s %03d %8d  %s",
                                "AFOS",
                                seqno,
                                mess->len,
                                mess->ident);
                /* Fatal, They will send it again */
                return -1;
        }

        return 0;
err:
        unrecognizable++;
        uerror("Unrecognizable: %5s %03d %8d",
                        "AFOS",
                        seqno,
                        buf->cnt);
        return -1;
}

extern void toClients(timestampt arrival,
        unsigned seqno,
        const char *ident,
        unsigned len,
        const char *buf);

static void
afos_send_buf(xbuf *buf, int backoff)
{
        xbuf clone[1];
        afos_message mess[1];

        seqno++;

        clone_xbuf(buf,clone, backoff);

        if(get_afos_message(clone, mess) != 0)
                return;
        /* else, we got something */

        toClients(mess->arrival,
                seqno,
                mess->ident,
                (unsigned) mess->len, (char *)mess->msg);
}


int
scan_afos(xbuf *buf)
{
        int sent = 0;
        int ch;

        static enum afos_l_state {
            GND,
            Z_,
            Z_C_,
            Z_C_Z_,
            N_,
            N_N_,
            N_N_N_
        } lstate = GND;

        static enum afos_state {
                START,
                HEADER_SEEN,
                TRAILER_SEEN
        } state = START;

        while ((ch = nextc(buf)) != EOB)
        {
                switch (ch) {
                case 'Z':
                        if(lstate == Z_C_)
                                lstate = Z_C_Z_;
                        else
                                lstate = Z_;
                        break;

                case 'C':
                        if(lstate == Z_)
                                lstate = Z_C_;
                        else if(lstate == Z_C_Z_ )
                        {
                                /* Z_C_Z_C */
                                if(state == HEADER_SEEN)
                                {
                                        /* missing trailer */
                                        nnnn_missed++;
                                        udebug("Missing NNNN trailer");
                                        afos_send_buf(buf, 4);
                                        sent = 1;
                                }
                                justify_xbuf(buf, 4);
                                state = HEADER_SEEN;
                                lstate = GND;
                        }
                        break;
                case 'N':
                        switch (lstate) {
                        case N_N_N_ :
                                /* N_N_N_N */
                                if(state == TRAILER_SEEN)
                                {
                                        /* missing header */
                                        zczc_missed++;
                                        udebug("Missing ZCZC header");
                                }
                                afos_send_buf(buf, 0);
                                sent = 1;
                                justify_xbuf(buf, 0);
                                state = TRAILER_SEEN;
                                lstate = GND;
                                break;
                        case N_N_ :
                                lstate = N_N_N_;
                                break;
                        case N_ :
                                lstate = N_N_;
                                break;
                        default :
                                lstate = N_;
                                break;
                        } /* lstate switch */
                        break;
                default :
                        lstate = GND;
                        break;
                } /* ch switch */
        } /* input character loop */
        return sent;
}

/*
 *   Copyright 1995, University Corporation for Atmospheric Research
 *   See ../COPYRIGHT file for copying and redistribution conditions.
 */
/* $Id: afos_message.h,v 1.28 1996/09/13 16:48:07 davis Exp $ */
#ifndef _AFOS_MESSAGE_H_
#define _AFOS_MESSAGE_H_

#include "timestamp.h"
#include "xbuf.h"
#include "wmo_header.h"

/* size of the buffer for 9 character AFOS header */
#define MAX_AFOS_HLEN 16 /* actually, its 9 + 1 */
#define MAX_WMO_HLEN 24 /* actually, its 22 +1 */

/*
 * AFOS Message
 */
typedef struct {
        timestampt arrival; /* arrival time */
        char ident[MAX_AFOS_HLEN +MAX_WMO_HLEN + 4]; /* AFOS header and more */
        unsigned char *msg; /* the whole message */
        size_t len;
} afos_message;

extern void afos_stats(void);
extern int scan_afos(xbuf *buf);

#endif /* _AFOS_MESSAGE_H_ */