Re: 19991202: pqsurf SYN date code problem

Art,

It was a bug in pqsurf/surf_split.c,  line 33 should be:

        set_dtime(time, YY, GG, 0);

I'll attach surf_split.c.

Robb...




On Thu, 2 Dec 1999, Arthur A. Person wrote:

> Rob,
> 
> On Thu, 2 Dec 1999, Robb Kambic wrote:
> 
> > I don't know for sure what the product ID's look like in pqsurf. Could you
> > enter a command in pqsurf.conf like
> > 
> > WMO ^(...... .... ......)
> >     EXEC    /bin/touch      data/IDs/\1
> > 
> > This will show the product ID's, then it will be able to check if your
> > pqsurf.conf entries are correct or there is a problem with pqsurf. I
> > suspect that your pqsurf.conf entries need to be changed.
> 
> Here's the pqsurf.conf entry that worked:
> 
> WMO   ^(.*)
>       EXEC    /bin/touch "data/IDs/\1"
> 
> and here's some output.  The aaxx and bbxx stuff all have 0202mm while the
> rest have 0217mm.
> 
> aaxx 24507 020212 CCA
> aaxx 44203 020215 RRC
> aaxx 44207 020215 RRC
> aaxx 44218 020215 RRC
> aaxx 44230 020215 RRC
> aaxx 44265 020215 RRC
> aaxx 93004 020217
> aaxx 93023 020217
> aaxx 93069 020217
> aaxx 93110 020217
> aaxx 93186 020217
> aaxx 93196 020217
> aaxx 93245 020217
> aaxx 93292 020217
> aaxx 93305 020217
> aaxx 93309 020217
> aaxx 93334 020217
> aaxx 93373 020217
> aaxx 93404 020217
> aaxx 93420 020217
> aaxx 93439 020217
> aaxx 93498 020217
> aaxx 93515 020217
> aaxx 93527 020217
> aaxx 93546 020217
> aaxx 93597 020217
> aaxx 93615 020217
> aaxx 93678 020217
> aaxx 93709 020217
> aaxx 93773 020217
> aaxx 93781 020217
> aaxx 93781 020217
> aaxx 93800 020217
> aaxx 93805 020217
> aaxx 93831 020217
> aaxx 93845 020217
> aaxx 93896 020217
> aaxx 93909 020217
> aaxx 93929 020217
> aaxx 93947 020217
> aaxx 93987 020217
> aaxx 93994 020217
> aaxx 94120 020217 RRE
> aaxx 94150 020217 RRB
> aaxx 94238 020217 RRC
> aaxx 94326 020217 RRF
> aaxx 95322 020217 RRD
> bbxx 46061 020217 RRD
> bbxx 46132 020217
> bbxx 46181 020217
> bbxx 46185 020217
> bbxx 46204 020217
> bbxx 46207 020217
> bbxx 46208 020217
> metar BIKF 021700 RRA
> metar EGAE 021720 RRB
> metar EGHH 021720 RRB
> metar EGMC 021720 RRB
> metar EGNH 021720 RRB
> metar EGNJ 021720 RRB
> metar EGPE 021720 RRB
> metar EGPN 021720 RRB
> metar EGPO 021720 RRB
> metar EGPO 021720 RRB
> metar EGSH 021720 RRB
> metar EHDL 021720 RRB
> metar EHEH 021720 RRB
> metar EHGR 021720 RRB
> metar EHKD 021720 RRB
> metar EHLW 021720 RRB
> metar EHSB 021720 RRB
> metar EHTW 021720 RRB
> metar EHVB 021720 RRB
> metar EICK 021730 RRB
> metar EIDW 021730 RRA
> metar EIKN 021730
> metar EINN 021730 RRC
> metar ENAL 021720 RRB
> metar ENCN 021720 RRB
> metar ENHD 021720 RRB
> metar ENRY 021720 RRB
> metar ENTC 021720 RRB
> metar ESOK 021720 RRA
> metar HKNA 021700 RRA
> metar K3HT 021500 CCI
> metar K3HT 021700 RRG
> metar KADU 021731
> metar KAFJ 021736
> metar KAIO 021730
> metar KAIZ 021736
> metar KALM 021733
> metar KANW 021735
> metar KAPF 021736
> metar KARG 021736
> metar KATS 021733
> metar KATS 021733
> metar KAUH 021735
> metar KAUO 021736
> metar KAWG 021730
> metar KAXA 021730
> metar KBCB 021730
> metar KBHB 021736
> metar KBID 021736
> metar KBIE 021733
> metar KBJI 021736
> metar KBKX 021736
> metar KBLM 021735
> metar KBNW 021730
> metar KBTP 021736
> metar KBVX 021736
> metar KCAV 021730
> metar KCBF 021731
> metar KCCY 021730
> metar KCGZ 021736
> metar KCIN 021730
> metar KCIU 021736
> metar KCNC 021730
> metar KCSQ 021730
> metar KDEH 021731
> metar KDNS 021730
> metar KEBS 021731
> metar KEOK 021731
> metar KFAT 021700 RRC
> metar KFET 021733
> metar KFFL 021730
> metar KGEG 021700 RRC
> metar KHDE 021733
> metar KHDE 021733
> metar KICL 021731
> metar KIKV 021731
> metar KLRJ 021731
> metar KLXN 021733
> metar KMCC 021733
> metar KMSV 021735
> metar KMUT 021730
> metar KMXO 021730
> metar KOGA 021733
> metar KOLZ 021731
> metar KONL 021733
> metar KORC 021730
> metar KOXV 021731
> metar KPIR 021700 RRM
> metar KRDK 021730
> metar KSDA 021730
> metar KSGF 021700 RRC
> metar KSHL 021731
> metar KSLB 021731
> metar KSRR 021735
> metar KSVC 021735
> metar KTNU 021731
> metar KVDI 021735
> metar KWWD 021735
> metar LFBD 021730
> metar LFBO 021730
> metar LFKB 021730
> metar LFKJ 021730
> metar LFLL 021730
> metar LFML 021730
> metar LFMN 021730
> metar LFMN 021730
> metar LFPB 021730
> metar LFPG 021730
> metar LFPO 021730
> metar LFSB 021730
> metar LGAD 021720 RRA
> metar LGAL 021720 RRA
> metar LGAT 021720 RRA
> metar LGEL 021720 RRA
> metar LGKL 021720 RRA
> metar LGKO 021720 RRA
> metar LGKR 021720 RRA
> metar LGLM 021720 RRA
> metar LGMT 021720 RRA
> metar LGPZ 021720 RRA
> metar LGRP 021720 RRA
> metar LGRX 021720 RRA
> metar LGSA 021720 RRA
> metar LGSM 021720 RRA
> metar LGTS 021720 RRA
> metar LHBP 021730
> metar LHBP 021730 CCA
> metar LPFR 021730
> metar LPPR 021730
> metar LPPT 021730
> metar MTPP 021700
> metar NZCM 021700 RRC
> metar PKMJ 021600
> metar RKSS 021730
> metar SARL 021700 RRB
> metar SARP 021700 RRD
> metar SBGL 021700 RRB
> metar SBGL 021700 RRB
> metar SBRF 021700 RRB
> metar SBTT 021700 RRB
> metar SBUG 021700 RRE
> metar SEQU 021700 RRC
> metar UMMS 021730
> metar YPAD 021700 RRB
> metar YPAD 021700 RRD
> sao SA WHP 021731
> sao SA WHY 021700 RRC
> sao SA WIA 021700 RRC
> sao SA WIG 021700 RRC
> sao SA WIP 021700 RRC
> sao SA WIU 021700 RRC
> sao SA WNL 021729
> sao SA WQS 021728
> sao SA WZO 021700 RRC
> sao SA XDE 021700 RRD
> sao SA XHF 021700 RRD
> sao SA XTV 021700 RRD
> speci CWUW 021729
> speci CYGQ 021725
> speci CYGR 021731
> speci CYKL 021731
> speci CYOW 021729
> speci CYOW 021732
> speci CYQT 021732
> speci CYUT 021726 CCA
> speci CYWA 021729
> speci CZBF 021728
> speci KAPC 021733
> speci KBNO 021731
> speci KCSM 021731
> speci KDEW 021730
> speci KFAT 021731
> speci KGEG 021733
> speci KINK 021731
> speci KINK 021732
> speci KKLS 021733
> speci KLKV 021730
> speci KLKV 021732
> speci KLKV 021733
> speci KRND 021730
> speci KSGF 021733
> speci KSSC 021730
> speci KWRL 021732
> speci PACD 021732
> speci PACD 021734
> speci PAEN 021731
> speci PHHI 021733
> speci PHHI 021734
> speci PHLI 021730
> speci PHLI 021732
> speci RJSM 021731
> speci SARE 021731
> speci SEQU 021731
> 
> 
>                             Art.
> 
> Arthur A. Person
> Research Assistant, System Administrator
> Penn State Department of Meteorology
> email:  address@hidden, phone:  814-863-1563
> 

===============================================================================
Robb Kambic                                Unidata Program Center
Software Engineer III                      Univ. Corp for Atmospheric Research
address@hidden             WWW: http://www.unidata.ucar.edu/
===============================================================================

/*
 *   Copyright 1993, University Corporation for Atmospheric Research
 *   See ../COPYRIGHT file for copying and redistribution conditions.
 */
/* $Id: surf_split.c,v 1.30 1999/12/02 23:21:26 rkambic Exp $   */

#include <ldmconfig.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include "ldm.h"
#include "ulog.h"
#include "wmo_header.h"
#include "tokens.h"
#include "xbuf.h"
#include "surface.h" /* wind_units_t, CALL_SIGN_LEN */

#include "md5.h"

static double md5ctx[16]; /* 88 would be big enough */
static MD5_CTX *md5ctxp = (MD5_CTX *)md5ctx;

static int
get_yygg(xbuf *buf, dtime *time)
{
        int status;
        int YY = -1;
        int GG = -1;

        if((status = dget_wnum(buf, &YY, 2)) < 0) return status;
        if((status = dget_num(buf, &GG, 2)) < 0) return status;

        set_dtime(time, YY, GG, 0);
        return status;
}

/* For METAR, check if the HHMMZ time string is present */
static int
whas_yyggZ(xbuf *buf)
{
        int ch;

        /* skip white space */
        do{
                ch = nextc(buf);
        }while((isascii(ch) && !isgraph(ch)));
        unnextc(buf,ch);

        if(buf->cnt < 5)
                return 0; /* not enough characters */
        if(buf->get[4] != 'Z'
                         || !isdigit(buf->get[3])
                         || !isdigit(buf->get[2])
                         || !isdigit(buf->get[1])
                         || !isdigit(buf->get[0]))
                return 0;
        return 1; /* passed */
}

/* For METAR, check if "NIL" */
static int
has_NIL(xbuf *buf)
{
        char nilstr[]  = "NIL";
        char *np = (char *)&buf->base[buf->bufsiz - 1 - (sizeof(nilstr) -1 -1)];

        if(strncmp(np, nilstr, sizeof(nilstr) -1) == 0)
                return 1;
        return 0;
}

/* For METAR, get the bulletin time, if possible */
static void
get_wyyggZ(xbuf *buf, dtime *time)
{
        int ch;
        if(!whas_yyggZ(buf))
                return;
        (void)get_yygg(buf, time);
        ch = nextc(buf); /* eat the 'Z' */
        return;
}


/*
 *      Takes a WMO format product which is a
 *   SAO, SYNOP, SHIP, METAR, or SPECI message, splits it into
 *   individual observations. The observations are each encapsulated in a
 *   new product which inherits most of its description from the
 *   original product.
 *  The new product pkey is derived from the observation type
 *   and has the following form:
 *
 *              SAO -   "sao tt ccc ddhhmm"
 *                      where:
 *                              tt is SA, SP or RS 
 *                              ccc is the station ID like SFO, LXV, etc
 *                              ddhhmm is the time stamp.
 *
 *              SYNOP - "aaxx nnnnn ddhhmm"
 *                      where:
 *                              nnnnn is the WMO station id (5 digit number)
 *
 *              SHIP -  "bbxx c* ddhhmm"
 *                      where:
 *                              c* is the call sign
 *
 *              METAR - "metar cccc ddhhmm"
 *                      where:
 *                              cccc is the call sign
 *
 *              SPECI - "speci cccc ddhhmm" 
 *
 *  The new product sequence number is original sequence number times 1000
 *   plus the sequence of the individual observation within the product.
 *
 *      'doit' is called on each of the new products. It is presumed
 * this function return  zero upon success.
 * 
 *  Returns the number of successful calls to 'doit', eg, the
 *  number of splits. Returns -1 on error.
 */
int
surf_split(const prod_info *infop, const void *datap,
                int (*doit)(const prod_info *, const void *))
{
        int action = -1;
        wmo_header_t hdr;
        message_type_t mtype;
        dtime dt;
        xbuf buf[1];
        unsigned char dbuf[8192]; /* TODO */
        int nsplit = 0;

        enum {
                SURFACE_BOGUS ,
                AAXX,
                US_AAXX,
                BBXX,
                SAO,
                sMETAR,
                sSPECI
        } subtype = SURFACE_BOGUS;

        hdr.time = &dt;

        if(infop->sz > sizeof(dbuf))
                return -1; /* TODO: too big */

        memcpy(dbuf, datap, infop->sz);

        if( cbuftoxbuf(buf, (unsigned char *)dbuf,
                        infop->sz) == NULL)
                return -1;
        
        skipline(buf, 4); /* SOH */
        skipline(buf, 12); /* start */

        if( get_wmo_header(buf, &hdr) == NULL)
        {
                return -1;
        } 
#if DEBUG
        fputs("\t", stderr);
        fprint_wmo_header(stderr, &hdr);
        fputs("\n", stderr);
#endif

        mtype = decode_type(hdr.TT,hdr.AA);
        
        /* #### */
        {
        char cbuf[8];
        int digit;
        dtime time;
        wind_units_t wind_units = WIND_UNAVAIL;

        time = *hdr.time; /* default the ob time to the time in the header */

        /* delve into section 0 */

        switch(mtype) {
        case SYNOP :
                if(get_wstr(buf, cbuf, 1) < 0 ) return -1;
                if(cbuf[0] == 'A')
                {
                        subtype = AAXX;
                        if(get_str(buf, &cbuf[1], 3) < 0 ) return -1;
                        if( cbuf[3] != 'X' )
                        {
                                /* punt */
                                uerror("surface_split: Unknown type: %s\n", 
cbuf);
                                return 0;
                        }
                        if(get_yygg(buf, &time) < 0 ) return -1; /* YYGG */
                        if(dget_num(buf, &digit, 1) < 0 ) return -1; /* isubw */
                        if(digit >= 0 && digit <= 4) wind_units = 
(wind_units_t)digit;
                }
                else if(isascii(cbuf[0]) && isdigit(cbuf[0])) /* US Stations 
7NNNN */
                {
                        unnextc(buf,cbuf[0]);
                        subtype = US_AAXX;
                        /* 
                         * Some US reports leave off AAXX YYGGisubw, so we use 
the
                         * time from the wmo header. 
                         */
                        wind_units = KNOTS;
                }
                else
                {
                        unnextc(buf,cbuf[0]);
                        return 0; /* ?? */
                }
                break;
        case SHIP :
                if(get_wstr(buf, cbuf, 4) < 0 ) return -1;
                if(cbuf[0] == 'B')
                {
                        if( cbuf[3] != 'X' )
                        {
                                /* punt */
                                uerror("surface_split: Unknown type: %s\n", 
cbuf);
                                return 0;
                        }
                        subtype = BBXX;
                        /* get time below */
                }
                else
                {
                        unnextc(buf,cbuf[0]);
                        return 0;
                }
                break;
        case METAR :
                if(whasSTR(buf, "METAR"))
                {
                        subtype = sMETAR;
                        get_wyyggZ(buf, &time);
                }
                else
                        subtype = SAO; /* may actually be a METAR, check below 
*/
                break;  
        case SPECI :
                if(whasSTR(buf, "SPECI"))
                {
                        subtype = sSPECI;
                        get_wyyggZ(buf, &time);
                }
                break;  
        default :
                uerror("surface_split: Can't handle %s", 
                        sMessage_type(mtype) );
                return -1;
        }

        { /* while block */
        static char newkey[KEYSIZE];
        xbuf subbuf[1];
        prod_info newinfo = *infop;
#define MAX_SURF_LEN 511
#undef MIN
#define MIN(a,b) ((a) <= (b) ? (a) : (b))
        char pbuf[MAX_SURF_LEN + 1];
        int l1, l2;
        static char ident[CALL_SIGN_LEN+1];
        static char type[4];
        u_int subseq = infop->seqno * 1000; 
        unsigned char *pp;

        while( get_weqxbuf(buf, subbuf) > 0 )
        {
                (void)memset(newkey,0,KEYSIZE);
                (void)memset(pbuf,0,MAX_SURF_LEN + 1);
                (void)memset(ident,0,CALL_SIGN_LEN+1);
                pp = subbuf->base;

                switch(subtype) {
                case AAXX :
                case US_AAXX :
                        strcpy(newkey, "aaxx ");
                        strcpy(pbuf, "AAXX");
                        sprintf(&pbuf[strlen(pbuf)], " %02d%02d%1d\r\r\n",
                                time.mday, time.hour, (int)wind_units);
                                        /* WMO station no. */
                        if(get_wstr(subbuf, ident, 5) < 0)
                                continue;
                        strcat(newkey, ident);
                        break;
                case BBXX :
                        strcpy(newkey, "bbxx ");
                        strcpy(pbuf, "BBXX\r\r\n");
                        /* call sign */
                        if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        strcat(newkey, ident);
                        if(get_yygg(subbuf, &time) < 0) continue; /* YYGG */
                        break;
                case sSPECI :
                        /* call sign */
                        if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        if(strcmp(ident, "SPECI") == 0)
                        {
                                /* They package each ob with a tag */
                                pp = (subbuf->get +1);
                                if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        }
                        if(!whas_yyggZ(subbuf))
                        {
                                /* Have to insert the date */
                                sprintf(pbuf, "SPECI\r\r\n%s %02d%02dZ ",
                                        ident, time.hour, time.min);
                                pp = subbuf->get;
                        }
                        else
                                strcpy(pbuf, "SPECI\r\r\n");
                        strcpy(newkey, "speci ");
                        strcat(newkey, ident);
                        break;
                case sMETAR :
                        if(has_NIL(subbuf))
                                continue;
                        /* call sign */
                        if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        if(strcmp(ident, "METAR") == 0)
                        {
                                /* They package each ob with a tag */
                                pp = (subbuf->get +1);
                                if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        }
                        if(!whas_yyggZ(subbuf))
                        {
                                /* Have to insert the date */
                                sprintf(pbuf, "METAR\r\r\n%s %02d%02dZ ",
                                        ident, time.hour, time.min);
                                pp = subbuf->get;
                        }
                        else
                                strcpy(pbuf, "METAR\r\r\n");
                        strcpy(newkey, "metar ");
                        strcat(newkey, ident);
                        break;
                case SAO :
                        /* call sign */
                        if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                continue;
                        if(hdr.AA[0] == 'U' && hdr.AA[1] == 'S'
                                        && strlen(ident) == 6)
                        {
                                /* skip 6 char US "AFOS code" */
                                if(get_wstr(subbuf, ident, CALL_SIGN_LEN) < 0)
                                        continue;
                        }
                                
                        /* SA, SP, RS, USP or XP */
                        if(get_wstr(subbuf, type, 3) < 0)
                                continue;
                        if((type[0] == 'S'
                                         && (type[1] == 'A' || type[1] == 'P'))
                                || (type[0] == 'R' && type[1] == 'S')
                                || (type[0] == 'U' && type[1] == 'S'
                                         && type[2] == 'P')
                                || (type[0] == 'X' && type[1] == 'P')
                                || (type[0] == 'T' &&
                                         (type[1] == 'A' || type[1] == 'S'))
                                )
                        {
                                strcpy(newkey, "sao ");
                                strcat(newkey, type);
                                strcat(newkey, " ");
                                strcat(newkey, ident);
                        } 
                        else if(isdigit(type[0]) && isdigit(type[1]))
                        {
                                /* it is a METAR really */
                                subtype = sMETAR;
                                strcpy(newkey, "metar ");
                                strcat(newkey, ident);
                                strcpy(pbuf, "METAR\r\r\n");
                        }
                        else
                                continue; /* don't know what it is, "NIL=" */
                        break;
                }

                /* safety net */
                if(strlen(ident) == 0)
                {
                        continue;
                }
                /* else */

                sprintf(&newkey[strlen(newkey)], " %02d%02d%02d",
                        time.mday, time.hour, time.min);
                if(hdr.retransmit != ORIGINAL)
                        sprintf(&newkey[strlen(newkey)], " %s",
                                sRetransmit(&hdr));
                newinfo.ident = newkey;
                newinfo.seqno = ++subseq;

                l1 = strlen(pbuf);
                l2 = MIN(MAX_SURF_LEN - l1 - 4, subbuf->bufsiz - (pp - 
subbuf->base));
                /* N.B.: silent truncation */
                strncat(pbuf, (char *)pp, l2 );
                strcat(pbuf,"=\r\r\n");

                newinfo.sz = l1 + l2 + 4;

#if DEBUG
                fprintf(stderr,"\t\t%s\n", newinfo.ident);
#endif
                
#if PRINT
                {
                        char *cp = pbuf;
                        char *end =  &cp[newinfo.sz];
                        while(cp < end)
                        {
                                putc(*cp, stderr);
                                cp++;
                        }
                }
                putc('\n', stderr);
#endif

                MD5Init(md5ctxp);
                MD5Update(md5ctxp, (const unsigned char *)pbuf, newinfo.sz);
                MD5Final(newinfo.signature, md5ctxp);
                
                /*
                 * process the single ob in the requested fashion
                 */
                if((*doit)(&newinfo, pbuf) == 0)
                        nsplit++;

        } /* end while */

#if PRINT
                putc('\n', stderr);
#endif
        } /* end while block */
        } /* end #### block */

        return nsplit;
}