Re: bufr reading

address@hidden wrote:
> Thanks for the link. Although I'm sure you've already seen this
> document, I thought I would pass it along anyway -
> http://www.wmo.ch/pages/prog/www/WDM/Guides/Guide-binary-1B.html. 
>
> The example I sent you is a DART/Tsunameter message.
>
> I believe there was some miscommunication on my part. What I was trying
> to ask - is there a method in your BUFR libraries that will allow me to
> easily apply the time increments to the appropriate data. I've been
> trying to figure out a way to use the SequenceDS object created in
> processBufrMessageDataset in the BufrDataDump program you sent me. I
> need to convert the data into a format (platform id, observation date,
> location, depth, value) that can be ingested into the OSMC. For this
> particular message, we're only interested in the water column height and
> the observations should look like: 
>
> platform id     observation date        location                                
>         depth   value
> 55013           7/12/2009 18:15:00      I get the location from metadata        
>         0
> 4919.7451171875
> 55013           7/12/2009 18:30:00                                              
>         0       4919.68310546875
> 55013           7/12/2009 18:45:00                                              
>         0       4919.6240234375
> 55013           7/12/2009 19:00:00                                              
>         0       4919.5703125
>
> The problem I'm having is that for the DART BUFR messages (or any
> message where the following occurs) the increments occur in conjunction
> with the replication. Using the SequenceDs object,
> I'm not sure how to determine which data to apply the increments. Do you
> have any suggestions and/or recommendations?
>
> Thanks!!
> Michelle
>   
looking more closely at the BUFR.ORIG file you sent, there are actually 
two kinds of messages in there, Tsunameter data (template 3-6-30) and 
Surface Data (template 3-6-28), heres a dump of the two along with an 
example:

1)

BUFR edition 4 time= Sat Jul 11 18:13:13 MDT 2009 wmoHeader=964IOZK01 
AMMC 130013 hash=[0xd1c00f4d] listHash=[0xffffc63d] (-14787)
  Category= 31.7.112=Oceanographic data / Tsunameter data
  Center= 1.0 (Melbourne)
  Table= 0.13.0
   Table B= wmoTable= wmo.v13.composite localTable= none mode=wmoOnly
   Table D= wmoTable= 
resource:/resources/bufrTables/wmo/TableD-121509.csv localTable= none
 DDS nsubsets=6 type=0xc0 isObs=true isCompressed=true
 startPos=31 len=140 endPos=171 dataStart=73 dataLen=94 dataEnd=167
   3-6-30  : (Sequence for representation of DART buoy standard hourly 
report)
     3-6-27  : (Sequence for representation of DART buoy 
identification  transmitter ID  type of tsunameter and the time the 
message is transmitted to the ground system)
       0-1-5   : Buoy/platform identifier
       0-1-52  : Platform transmitter ID
       0-2-47  : Deep-ocean tsunameter type
       3-1-11  :
         0-4-1   : Year
         0-4-2   : Month
         0-4-3   : Day
       3-1-13  :
         0-4-4   : Hour
         0-4-5   : Minute
         0-4-6   : Second
     3-6-29  : (Sequence for representation of tsunameter sampling 
information for water column heights in the time series report)
       0-25-170: sampling interval (time)
       0-25-171: sample averaging period
       0-25-172: Number of samples
     1-11-000: replication
     0-31-1  : Delayed descriptor replication factor
     0-33-2  : Quality information
     3-1-11  :
       0-4-1   : Year
       0-4-2   : Month
       0-4-3   : Day
     3-1-13  :
       0-4-4   : Hour
       0-4-5   : Minute
       0-4-6   : Second
     0-25-25 : Battery voltage
     0-25-25 : Battery voltage
     0-25-26 : Battery voltage (large range)
     0-22-185: BPR transmission count
     0-4-15  : Time increment
     0-4-65  : Short time increment
     1-01-004: replication
     0-22-182: Water column height

 CDM Nested Table=
   0-1-5: Buoy/platform identifier units=Numeric scale=0 refVal=0 nbits=17
   0-1-52: Platform transmitter ID units=Code table enum nbits=3
   0-2-47: Deep-ocean tsunameter type units=Code table enum nbits=7
   0-4-1: Year units=Year scale=0 refVal=0 nbits=12
   0-4-2: Month units=Month scale=0 refVal=0 nbits=4
   0-4-3: Day units=Day scale=0 refVal=0 nbits=6
   0-4-4: Hour units=Hour scale=0 refVal=0 nbits=5
   0-4-5: Minute units=Minute scale=0 refVal=0 nbits=6
   0-4-6: Second units=Second scale=0 refVal=0 nbits=6
   0-25-170: sampling interval (time) units=Second scale=0 refVal=0 
nbits=10
   0-25-171: sample averaging period units=Second scale=0 refVal=0 nbits=10
   0-25-172: Number of samples units=Numeric scale=0 refVal=0 nbits=10
   1-11-0: Replication count=0 replicationCountSize=8
     0-33-2: Quality information units=Code table enum nbits=2
     0-4-1: Year units=Year scale=0 refVal=0 nbits=12
     0-4-2: Month units=Month scale=0 refVal=0 nbits=4
     0-4-3: Day units=Day scale=0 refVal=0 nbits=6
     0-4-4: Hour units=Hour scale=0 refVal=0 nbits=5
     0-4-5: Minute units=Minute scale=0 refVal=0 nbits=6
     0-4-6: Second units=Second scale=0 refVal=0 nbits=6
     0-25-25: Battery voltage units=V scale=1 refVal=0 nbits=9
     0-25-25: Battery voltage units=V scale=1 refVal=0 nbits=9
     0-25-26: Battery voltage (large range) units=V scale=1 refVal=0 
nbits=12
     0-22-185: BPR transmission count units=Numeric scale=0 refVal=0 
nbits=10
     0-4-15: Time increment units=Minute scale=0 refVal=-2048 nbits=12
     0-4-65: Short time increment units=Minute scale=0 refVal=-128 nbits=8
     1-1-4: Replication count=4
       0-22-182: Water column height units=m scale=3 refVal=0 nbits=23

time =0 secs since 2009-07-12T00:13:13Z
 Buoy/platform identifier =55013
 Platform transmitter ID =1 CodeTable 0-1-52
 Deep-ocean tsunameter type =1 CodeTable 0-2-47
 Year =2009 Year
 Month =7 Month
 Day =12 Day
 Hour =0 Hour
 Minute =13 Minute
 Second =13 Second
 sampling interval (time) =900 Second
 sample averaging period =15 Second
 Number of samples =1
 seq1 =
   {
     Quality information =  "Data not suspect" CodeTable 0-33-2
     Year =2009 Year
     Month =7 Month
     Day =12 Day
     Hour =18 Hour
     Minute =15 Minute
     Second =0 Second
     Battery voltage =14.7 V
     Battery voltage-1 =4.1 V
     Battery voltage (large range) =26.0 V
     BPR transmission count =1
     Time increment =-15.0 Minute
     Short time increment =15.0 Minute
     Water column height =
       {4919.745, 4919.683, 4919.624, 4919.5703} m
   } obs.seq1(0)

2)

BUFR edition 4 time= Sat Jul 11 18:19:18 MDT 2009 wmoHeader=699IOZK10 
AMMC 130019 hash=[0xd1be6296] listHash=[0xffffc63b] (-14789)
  Category= 31.0.112=Oceanographic data / Surface observation
  Center= 1.0 (Melbourne)
  Table= 0.13.0
   Table B= wmoTable= wmo.v13.composite localTable= none mode=wmoOnly
   Table D= wmoTable= 
resource:/resources/bufrTables/wmo/TableD-121509.csv localTable= none
 DDS nsubsets=1 type=0x80 isObs=true isCompressed=false
 startPos=206 len=70 endPos=276 dataStart=248 dataLen=24 dataEnd=272
   3-6-28  : (Sequence for representation of time of observation and 
DART buoy position daily report)
     3-6-27  : (Sequence for representation of DART buoy 
identification  transmitter ID  type of tsunameter and the time the 
message is transmitted to the ground system)
       0-1-5   : Buoy/platform identifier
       0-1-52  : Platform transmitter ID
       0-2-47  : Deep-ocean tsunameter type
       3-1-11  :
         0-4-1   : Year
         0-4-2   : Month
         0-4-3   : Day
       3-1-13  :
         0-4-4   : Hour
         0-4-5   : Minute
         0-4-6   : Second
     3-1-11  :
       0-4-1   : Year
       0-4-2   : Month
       0-4-3   : Day
     3-1-13  :
       0-4-4   : Hour
       0-4-5   : Minute
       0-4-6   : Second
     3-1-21  :
       0-5-1   : Latitude (high accuracy)
       0-6-1   : Longitude (high accuracy)

 time =0 secs since 2009-07-12T00:19:18Z
 Buoy/platform identifier =56003
 Platform transmitter ID =0 CodeTable 0-1-52
 Deep-ocean tsunameter type =1 CodeTable 0-2-47
 Year =2009 Year
 Month =7 Month
 Day =12 Day
 Hour =0 Hour
 Minute =19 Minute
 Second =18 Second
 Year-1 =2009 Year
 Month-1 =7 Month
 Day-1 =12 Day
 Hour-1 =6 Hour
 Minute-1 =47 Minute
 Second-1 =40 Second
 Latitude (high accuracy) =-15.019232 degrees_north
 Longitude (high accuracy) =118.004105 degrees_east


the first one has the nested sequence, the second only seems to have 
lat/lon.

the nested sequence is rather awkward to process, but you just have to 
follow it in and extract the data. Ive attached a quick and dirty 
example program that should get you started. output looks like:

Dump D:/work/michelle/TimeIncr.bufr
Message 0 header=964IOZK01 AMMC 130013
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=18, min=15, 
sec=0, incr=-15, incrS=15, value=4919.745 4919.683 4919.624 4919.5703 }
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=19, min=15, 
sec=0, incr=-15, incrS=15, value=4919.5303 4919.4854 4919.4443 4919.413 }
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=20, min=15, 
sec=0, incr=-15, incrS=15, value=4919.3853 4919.3604 4919.3423 4919.333 }
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=21, min=15, 
sec=0, incr=-15, incrS=15, value=4919.3354 4919.3394 4919.352 4919.37 }
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=22, min=15, 
sec=0, incr=-15, incrS=15, value=4919.4043 4919.4424 4919.4834 4919.5293 }
Extract{platformId=55013.0, year=2009, month=7, day=12, hour=23, min=15, 
sec=0, incr=-15, incrS=15, value=4919.579 4919.637 4919.6973 4919.7603 }
......


let me know how it goes

/*
 * Copyright (c) 1998 - 2009. University Corporation for Atmospheric 
Research/Unidata
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package ucar.nc2.iosp.bufr.writer;

import ucar.nc2.iosp.bufr.Message;
import ucar.nc2.iosp.bufr.BufrIosp;
import ucar.nc2.iosp.bufr.MessageScanner;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.SequenceDS;
import ucar.nc2.dataset.VariableDS;
import ucar.nc2.dataset.StructureDS;
import ucar.nc2.util.Indent;
import ucar.nc2.Variable;
import ucar.nc2.NetcdfFile;
import ucar.ma2.*;
import ucar.unidata.io.RandomAccessFile;

import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;

/**
 * reads a file with BUFR messages in it and prints out the data values.
 *
 * @author caron
 * @since Dec 17, 2009
 */
public class BufrDataDump {

  private PrintStream out;
  private Indent indent = new Indent(2);

  /**
   * Open file as a stream of BUFR messages and print data
   *
   * @param filename open this file
   * @param os       print to here
   * @throws IOException on IO error
   */
  public BufrDataDump(String filename, OutputStream os) throws IOException {
    out = new PrintStream(os);
    out.format("Dump %s%n", filename);
    indent.setIndentLevel(0);

    try {
      scanBufrFile(filename);

    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e.getMessage());
    }

    out.flush();

  }

  // open the file and extract BUFR messages
  public void scanBufrFile(String filename) throws Exception {
    int count = 0;
    RandomAccessFile raf = null;
    try {
      raf = new RandomAccessFile(filename, "r");

      MessageScanner scan = new MessageScanner(raf);
      while (scan.hasNext()) {
        Message m = scan.next();
        if (m == null) continue;
        out.format("%sMessage %d header=%s%n", indent, count++, m.getHeader());
        processBufrMessageAsDataset(scan, m);
      }
    } finally {
      if (raf != null)
        raf.close();
    }
  }

  // convert one message ino a NetcdfDataset and print data
  private void processBufrMessageAsDataset(MessageScanner scan, Message m) 
throws Exception {
    byte[] mbytes = scan.getMessageBytes(m);
    NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, 
"ucar.nc2.iosp.bufr.BufrIosp");
    NetcdfDataset ncd = new NetcdfDataset(ncfile);
    SequenceDS obs = (SequenceDS) ncd.findVariable(BufrIosp.obsRecord);
    StructureDataIterator sdataIter = obs.getStructureIterator(-1);
    //writeSequence(obs, sdataIter);
    extractFirst(sdataIter, new Extract());
  }

  private class Extract {
    double platformId;
    int year,month,day,hour,min,sec,incr,incrS;
    Array value;

    @Override
    public String toString() {
      return "Extract{" +
              "platformId=" + platformId +
              ", year=" + year +
              ", month=" + month +
              ", day=" + day +
              ", hour=" + hour +
              ", min=" + min +
              ", sec=" + sec +
              ", incr=" + incr +
              ", incrS=" + incrS +
              ", value=" + value +
              '}';
    }
  }

  // iterate through the observations
private void extractFirst(StructureDataIterator sdataIter, Extract result) 
throws IOException {
  while (sdataIter.hasNext()) {
    StructureData sdata = sdataIter.next();

    for (StructureMembers.Member m : sdata.getMembers()) {
      if (m.getName().equals("Buoy/platform identifier"))
        result.platformId = sdata.convertScalarDouble(m);

      else if (m.getDataType() == DataType.SEQUENCE) {
        ArraySequence data = (ArraySequence) sdata.getArray(m);
        extractNested(data.getStructureDataIterator(), result);
      }
    }
  }
}

  // iterate through the observations
private void extractNested(StructureDataIterator sdataIter, Extract result) 
throws IOException {
  while (sdataIter.hasNext()) {
    StructureData sdata = sdataIter.next();

    for (StructureMembers.Member m : sdata.getMembers()) {
      if (m.getName().equals("Year"))
        result.year = sdata.convertScalarInt(m);
      else if (m.getName().equals("Month"))
        result.month = sdata.convertScalarInt(m);
      else if (m.getName().equals("Day"))
        result.day = sdata.convertScalarInt(m);
      else if (m.getName().equals("Hour"))
        result.hour = sdata.convertScalarInt(m);
      else if (m.getName().equals("Minute"))
        result.min = sdata.convertScalarInt(m);
      else if (m.getName().equals("Second"))
        result.sec = sdata.convertScalarInt(m);
      else if (m.getName().equals("Time increment"))
        result.incr = sdata.convertScalarInt(m);
      else if (m.getName().equals("Short time increment"))
        result.incrS = sdata.convertScalarInt(m);
      else if (m.getName().equals("Water column height")) {
        result.value = sdata.getArray(m);
        out.format("%s%n", result.toString());
      }
    }
  }
}

  // iterate through the observations
  private void writeSequence(StructureDS s, StructureDataIterator sdataIter) 
throws IOException {
    indent.incr();
    int count = 0;
    while (sdataIter.hasNext()) {
      out.format("%sSequence %s count=%d%n", indent, s.getShortName(), count++);
      StructureData sdata = sdataIter.next();
      indent.incr();

      for (StructureMembers.Member m : sdata.getMembers()) {
        Variable v = s.findVariable(m.getName());

        if (m.getDataType().isString() || m.getDataType().isNumeric()) {
          writeVariable((VariableDS) v, sdata.getArray(m));

        } else if (m.getDataType() == DataType.STRUCTURE) {
          StructureDS sds = (StructureDS) v;
          ArrayStructure data = (ArrayStructure) sdata.getArray(m);
          writeSequence(sds, data.getStructureDataIterator());

        } else if (m.getDataType() == DataType.SEQUENCE) {
          SequenceDS sds = (SequenceDS) v;
          ArraySequence data = (ArraySequence) sdata.getArray(m);
          writeSequence(sds, data.getStructureDataIterator());
        }
      }
      indent.decr();
    }

    indent.decr();
  }

  private void writeVariable(VariableDS v, Array mdata) throws IOException {
    int count = 0;
    String name = v.getShortName();
    String units = v.getUnitsString();
    out.format("%svar='%s' units='%s' : ", indent, name, units);

    mdata.resetLocalIterator();
    while (mdata.hasNext()) {
      if (count++ > 0) out.format(",");

      if (v.getDataType().isNumeric()) {

        double val = mdata.nextDouble();
        if (v.isMissing(val)) { // check if missing
          out.format("missing");
        } else {
          out.format("%s", Double.toString(val));
        }

      } else {  // not numeric
        out.format("%s", mdata.next());
      }
    }
    out.format("%n");
  }

  public static void main(String[] args) throws IOException {
    new BufrDataDump("D:/work/michelle/TimeIncr.bufr", System.out);
    //new BufrDataDump("D:/work/michelle/DART.bufr", System.out);
  }

}