[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: thredds like dods server

Hi Fabrizio,

Yuan forwarded your email with the CatalogGen input and output files to me.
Looks like there are two metadata elements in your input file that CatalogGen would try to expand. The one where you try to crawl your OPeNDAP server (http://192.107.66.16:8080/thredds/content/dodsC/Dati/subdir/) is failing because CatalogGen expects responses like those returned by the C++ OPeNDAP servers not the thredds/dodsC/ server.
I'm not sure what is happening with the catalogGen that crawls a local directory. It doesn't seem to expand into anything which leads me to believe that the directory exists but it isn't finding anything in the directory that match the filter. Try removing the datasetFilter and datasetNamer elements and see what you get. 

Hope that helps,

Ethan


Fabrizio Paolucci wrote:


Hi Yuan,
this is the output file:


<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE catalog SYSTEM
"http://www.unidata.ucar.edu/projects/THREDDS/xml/InvCatalog.0.6.dtd";>
<catalog xmlns="http://www.unidata.ucar.edu/thredds";
xmlns:xlink="http://www.w3.org/1999/xlink"; name="My
Data" version="0.6">
 <dataset name="Example of cataloging"
dataType="Grid" serviceName="this">
   <service name="this" serviceType="DODS" base="" />
   <service name="merseaexm" serviceType="NetCDF"
base="http://mersea.santateresa.enea.it:8080/thredds/dodsC/";
/>
   <service name="http_estserver"
serviceType="NetCDF"
base="http://192.107.66.16/~jboss/Dati/"; />
   <service name="merseaexm_local"
serviceType="NetCDF"
base="file:///home/jboss/public_html/Dati/" />
   <service name="this" serviceType="DODS" base="" />
   <service name="mersea_test" serviceType="DODS"
base="http://192.107.66.16:8080/thredds/content/dodsC/Dati/";
/>
   <property name="internalService"
value="merseaexm_local" />
   <dataset name="mersea example aggregation Union"
urlPath="mersea_exm_union">
     <metadata metadataType="Aggregation">
       <aggregation aggType="Union"
serviceName="merseaexm_local">
<fileAccess serviceName="merseaexm_local" 
urlPath="NSCATstress.199702.nc" />
<fileAccess serviceName="merseaexm_local" 
urlPath="NSCATstress.199703.nc" />
</aggregation> 
     </metadata>
   </dataset>
   <dataset name="mersea example aggregation JoinNew"
urlPath="mersea_exm_joinnew">
     <metadata metadataType="Aggregation">
       <aggregation aggType="JoinNew"
dateFormat="yyyy/M/d:HH:mm:ss z"
serviceName="merseaexm_local" varName="time"
varType="int" varUnit="days since 0000-01-01 00:00">
<fileAccess coord="0000/01/01:00:00:00 GMT" 
serviceName="merseaexm_local"
urlPath="NSCATstress.199702.nc" />
<fileAccess coord="0000/02/01:00:00:00 GMT" 
serviceName="merseaexm_local"
urlPath="NSCATstress.199703.nc" />
</aggregation> 
     </metadata>
   </dataset>
   <dataset name="mersea example aggregation
JoinExisting" urlPath="mersea_exm_joinexsisting">
     <metadata metadataType="Aggregation">
       <aggregation aggType="JoinExisting"
serviceName="merseaexm_local" varName="lat">
<fileAccess serviceName="merseaexm_local" 
urlPath="NSCATstress.199702.nc" />
<fileAccess serviceName="merseaexm_local" 
urlPath="NSCATstress.199703.nc" />
</aggregation> 
     </metadata>
   </dataset>
   <dataset name="remote thredds server">
     <dataset name="Could not expand the
DatasetSource "our thredds server" at this location
"http://192.107.66.16:8080/thredds/content/dodsC/Dati/subdir/";.
IoException while checking that the location
references an OPeNDAP server."
serviceName="mersea_test" />
   </dataset>
   <dataset name="subdir/" serviceName="this">
     <dataset name="199706"
urlPath="subdir/NSCATstress.199706.nc" />
     <dataset name="199705"
urlPath="subdir/NSCATstress.199705.nc" />
     <dataset name="199611"
urlPath="subdir/NSCATstress.199611.nc" />
   </dataset>
   <dataset name="199706"
urlPath="NSCATstress.199706.nc" serviceName="this" />
   <dataset name="199705"
urlPath="NSCATstress.199705.nc" serviceName="this" />
   <dataset name="199704"
urlPath="NSCATstress.199704.nc" serviceName="this" />
   <dataset name="199703"
urlPath="NSCATstress.199703.nc" serviceName="this" />
   <dataset name="199702"
urlPath="NSCATstress.199702.nc" serviceName="this" />
   <dataset name="199611"
urlPath="NSCATstress.199611.nc" serviceName="this" />
 </dataset>
</catalog>

and this is the input file:

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE catalog SYSTEM
"http://www.unidata.ucar.edu/projects/THREDDS/xml/AggServerCatalog.dtd";>
<catalog xmlns="http://www.unidata.ucar.edu/thredds";
xmlns:xlink="http://www.w3.org/1999/xlink"; name="My
Data" version="0.6">
 <dataset name="Example of cataloging"
dataType="Grid" serviceName="this">
   <service name="this" serviceType="DODS" base="" />
   <service name="merseaexm" serviceType="NetCDF"
base="http://mersea.santateresa.enea.it:8080/thredds/dodsC/";
/>
   <service name="http_estserver"
serviceType="NetCDF"
base="http://192.107.66.16/~jboss/Dati/"; />
   <service name="merseaexm_local"
serviceType="NetCDF"
base="file:///home/jboss/public_html/Dati/" />
<dataset name="mersea example aggregation Union" 
urlPath="mersea_exm_union">
    <metadata metadataType="Aggregation">
     <aggregation aggType="Union"
serviceName="merseaexm_local" >
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199702.nc" />
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199703.nc" />
     </aggregation>
</metadata> </dataset> <dataset name="mersea example aggregation JoinNew" 
urlPath="mersea_exm_joinnew">
    <metadata metadataType="Aggregation">
<aggregation aggType="JoinNew" serviceName="merseaexm_local" varName="time" 
varType="int" varUnit="days since 0000-01-01 00:00"
dateFormat="yyyy/M/d:HH:mm:ss z" >
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199702.nc"
coord="0000/01/01:00:00:00 GMT" />
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199703.nc"
coord="0000/02/01:00:00:00 GMT" />
     </aggregation>
</metadata> </dataset> <dataset name="mersea example aggregation 
JoinExisting" urlPath="mersea_exm_joinexsisting">
    <metadata metadataType="Aggregation">
<aggregation aggType="JoinExisting" serviceName="merseaexm_local" varName="lat" > 
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199702.nc"  />
       <fileAccess serviceName="merseaexm_local"
urlPath="NSCATstress.199703.nc" />
</aggregation> </metadata> </dataset> 
 <property name="internalService"
value="merseaexm_local"/>
 <metadata metadataType="CatalogGenConfig">
                                 <catalogGenConfig type="Catalog">
                                         <datasetSource name="File crawling" 
type="Local"
structure="DirTree"
accessPoint="/home/jboss/public_html/Dati/">
                                                 <resultService name="this" 
serviceType="DODS"
base=""
accessPointHeader="/home/jboss/public_html/Dati/" />
                                                 <datasetFilter name="Accept netCDF 
files only"
type="RegExp" matchPattern="\.nc$" />
                                                 <datasetNamer name="Container" 
type="RegExp"
addLevel="true" matchPattern="([a-z0-9]+).nc$"
substitutePattern="$1" />
                                         </datasetSource>
                                 </catalogGenConfig>
    </metadata>
                                        
 </dataset>               
 <dataset name="remote thredds server" >
     <metadata metadataType="CatalogGenConfig">
       <catalogGenConfig type="Catalog">
         <datasetSource name="our thredds server"
type="DodsDir"
           structure="Flat"
accessPoint="http://192.107.66.16:8080/thredds/content/dodsC/Dati/subdir/";> 
           <resultService name="mersea_test"
serviceType="DODS"
base="http://192.107.66.16:8080/thredds/content/dodsC/Dati/"; accessPointHeader="http://192.107.66.16:8080/thredds/content/dodsC/Dati/"; 
             />

            <datasetFilter name="Accept netCDF files
only" type="RegExp"
             matchPattern="\.nc$"/>

           <datasetNamer name="mersea_data"
type="RegExp" addLevel="true" matchPattern="([a-z0-9]+).nc$" substitutePattern="$1" 
/>
</datasetSource> 
       </catalogGenConfig>
     </metadata>
 </dataset>
 </dataset>
</catalog> notes: 

We have testing with only few .nc files and
 the aggregation works !!
We have read in the mailing list that aggregation
can,t work....
In the input file for cataloggen the 'static' part  of
dataset tag is copied in the output file, but we don't
found this feature in the documentation.
We are testing functionalities and what's behaviour expected for our project and also to contribute for 
your better documentation, if you want.
Thanks for your support and answer!
Best regards

Fabrizio Paolucci, Leda Pecci
ENEA 

--
Ethan R. Davis                                Telephone: (303) 497-8155
Software Engineer                             Fax:       (303) 497-8690
UCAR Unidata Program Center                   E-mail:    address@hidden
P.O. Box 3000
Boulder, CO  80307-3000                       http://www.unidata.ucar.edu/
---------------------------------------------------------------------------