http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/stream-definitions ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/stream-definitions b/nexus-ingest/developer-box/stream-definitions new file mode 100644 index 0000000..0188a98 --- /dev/null +++ b/nexus-ingest/developer-box/stream-definitions @@ -0,0 +1,64 @@ +stream create --name ingest-mur --definition "scan-for-mur-granules: file --dir=/vagrant/data/mur/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=5184 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-mur-clim --definition "scan-for-mur-clim-granules: file --dir=/vagrant/data/mur-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=5184 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groov y | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^mur_(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^mur_(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis --definition "scan-for-modis-granules: file --dir=/vagrant/data/modis/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4096 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=sst,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=sst --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:/ //home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^.*(\d{7})\.,dateformat=yyyyDDD' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3_sst_4km' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-clim --definition "scan-for-modisclim-granules: file --dir=/vagrant/data/modis-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4096 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=sst,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 --environment=STORED_VAR_NAME=mean_sst | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groov y --variables='regex=^A(\\d{3})\\.,dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^A(\\d{3})\\.' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-avhrr --definition "scan-for-avhrr-granules: file --dir=/vagrant/data/avhrr/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=analysed_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | s et-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=AVHRR_OI_L4_GHRSST_NCEI' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-avhrr-clim --definition "scan-for-avhrr-clim-granules: file --dir=/vagrant/data/avhrr-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,META=analysed_sst_std,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=analysed_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nex us-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=AVHRR_OI_L4_GHRSST_NCEI_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name "ingest-ascatb-u" --definition "scan-for-ascat-granules: file --dir=/vagrant/data/ascatb/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=NUMROWS,NUMCELLS --sliceByDimension=NUMROWS | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_swath_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.winddirspeedtouv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp,U_OR_V=u,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/ vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ASCATB-L2-Coastal' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=10" + +stream create --name "ingest-ascatb-v" --definition "scan-for-ascat-granules: file --dir=/vagrant/data/ascatb/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=NUMROWS,NUMCELLS --sliceByDimension=NUMROWS | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_swath_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.winddirspeedtouv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp,U_OR_V=v,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/ vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ASCATB-L2-Coastal' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=10" + +stream create --name ingest-trmm --definition "file --dir=/vagrant/data/trmm/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=TRMM_3B42_daily_precipitation_V7,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --sc ript=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=TRMM_3B42_daily' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-aqua-aod-500 --definition "file --dir=/vagrant/data/modis_aqua_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MYD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-i d.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_AQUA_AOD_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-aqua-aod-16 --definition "file --dir=/vagrant/data/modis_aqua_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=16 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MYD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id. groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_AQUA_AOD_16' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-terra-aod-500 --definition "file --dir=/vagrant/data/modis_terra_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MOD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile -id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_TERRA_AOD_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-terra-aod-16 --definition "file --dir=/vagrant/data/modis_terra_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=16 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MOD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-i d.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_TERRA_AOD_16' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name "ingest-smap-sss" --definition "scan-for-smap-granules: file --dir=/vagrant/data/smap/ --mode=ref --pattern=*.h5 --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=0,1 --sliceByDimension=1 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=smap_sss,LATITUDE=lat,LONGITUDE=lon,TIME=row_time,GLBLATTR_DAY=REV_START_TIME,GLBLATTR_DAY_FORMAT=%Y-%jT%H:%M:%S.%f,READER=SWATHTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus -shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=SMAP_L2B_SSS' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-grace --definition "scan-for-grace-granules: file --dir=/vagrant/data/grace/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 --splitResult=true | join-with-static-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=152' | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=lwe_thickness,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | subtract180: tcpshell --command='python -u -m nexusxd.subtract180longitude' --bufferSize=1000000 | normalize-time: tcpshell --command='python -u -m nexusxd.normalizetimebeginningofmonth' --bufferSize=1000000 | tile-summ ary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=lwe_thickness --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_OCEAN' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-ecco --definition "ecco: file --dir=/vagrant/data/ecco/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=500 --splitResult=true | join-with-static-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=240' | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MXLDEPTH,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | normalize-time: tcpshell --command='python -u -m nexusxd.normalizetimebeginningofmonth' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=MXL DEPTH --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MXLDEPTH_ECCO_version4_release1' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + + +stream create --name "ingest-ccmp-u" --definition "scan-for-ccmp-granules: file --dir=/vagrant/data/ccmp/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.computespeeddirfromuv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=uwnd,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,META=vwnd,READER=GRIDTILE,TEMP_DIR=/tmp,WIND_U=uwnd,WIND_V=vwnd,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:// /home/vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75" + +stream create --name "ingest-ccmp-spd" --definition "scan-for-ccmp-granules: file --dir=/vagrant/data/ccmp/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=wind_speed --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy --variab les='salt=wind_speed' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0_WIND_SPEED' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75" + +stream create --name "ingest-ccmp-spd-clim" --definition "scan-for-ccmp-clim-granules: file --dir=/vagrant/data/ccmp/climatology --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=latitude,LONGITUDE=longitude,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_wind_speed --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-t ile-id.groovy --variables='salt=wind_speed' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^CCMP_Wind_Analysis_V02.0_L3.0_RSS_(\\d{3})_1day_clim_pixelMean' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0_WIND_SPEED_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75" + + + +stream create --name "ingest-measures-ssh-native" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=ssh_grids*.nc --maxMessages=1 | to-filename: transform --expression=payload.absolutePath | permute-dimensions: tcpshell --command='python -u -m nexusxd.callncpdq' --environment=DIMENSION_ORDER=Time:Latitude:Longitude | file-ref: transform --expression=\"new java.io.File(new java.lang.String(payload))\" | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180lon gitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name "ingest-measures-ssh-1x1monthly" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=avg-regrid1x1-ssh_grids*.nc --maxMessages=1 --fixedDelay=1 | to-filename: transform --expression=payload.absolutePath | permute-dimensions: tcpshell --command='python -u -m nexusxd.callncpdq' --environment=DIMENSION_ORDER=Time:Latitude:Longitude | file-ref: transform --expression=\"new java.io.File(new java.lang.String(payload))\" | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_e mpty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603_1x1Monthly' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name "ingest-measures-ssh-avg" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=avg-regrid1x1-ssh_grids*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:/// home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name "ingest-measures-ssh-clim" --definition "scan-for-measures-clim-granules: file --dir=/vagrant/data/measures_alt/climatology --mode=ref --pattern=ssh_grids*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=Latitude,LONGITUDE=Longitude,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --scri pt=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^ssh_grids_v1609_(\\d{3})_5day_clim_pixelMean,dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^ssh_grids_v1609_(\\d{3})_5day_clim_pixelMean' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + + +stream create --name "ingest-ice_shelf-1x1" --definition "scan-for-ice_shelf-granules: file --dir=/vagrant/data/ice_shelf/ --mode=ref --pattern=1x1regrid-ice_shelf_dh_v1.nc --maxMessages=1 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4 | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=72' | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=height_filt,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=height_filt --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id. groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ICE_SHELF_DH_V1_1x1Monthly' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-chl_a --definition "scan-for-chla-granules: file --dir=/vagrant/data/modis_aqua_chl --mode=ref --pattern=*.nc --maxMessages=2 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=chlor_a,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=chlor_a --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/ad d-time-from-granulename.groovy --variables='regex=^A(\\d{7}),dateformat=yyyyDDD' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_DAY_CHL_chlor_a_4km' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-modis-chl_a-clim --definition "scan-for-chla-clim-granules: file --dir=/vagrant/data/modis_aqua_chl/climatology --mode=ref --pattern=*.nc --maxMessages=2 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_chlor_a --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=fil e:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^A(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^A(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_DAY_CHL_chlor_a_4km_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + + + +stream create --name ingest-smapl3 --definition "file --dir=/vagrant/data/smapl3/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=sss_smap,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-sha red/set-dataset-name.groovy --variables='datasetname=SMAP_SSS_L3_MONTHLY_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + +stream create --name ingest-smapl3-clim --definition "file --dir=/vagrant/sync/data/smapl3-clim/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-day-atr: script --script=file:///home/vagrant/xd-n exus-shared/add-day-of-year-attribute.groovy --variables='regex=^sss_smap_L3m_clim_doy(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=SMAP_SSS_L3_MONTHLY_500_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042" + + +
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/user-init.sh ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/user-init.sh b/nexus-ingest/developer-box/user-init.sh new file mode 100755 index 0000000..d831fe8 --- /dev/null +++ b/nexus-ingest/developer-box/user-init.sh @@ -0,0 +1,17 @@ +homedir=$1 +scriptdir=`dirname $0` + +pushd $homedir + +mkdir nexus +pushd nexus +git init +popd + +mkdir xd-nexus-shared + +touch /tmp/xdcommand + +popd + +$scriptdir/user-update.sh $homedir \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/user-update.sh ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/user-update.sh b/nexus-ingest/developer-box/user-update.sh new file mode 100755 index 0000000..2648a99 --- /dev/null +++ b/nexus-ingest/developer-box/user-update.sh @@ -0,0 +1,86 @@ +scriptdir=`dirname $0` + +if [ -z "$1" ]; then + homedir="/home/vagrant" +else + homedir=$1 +fi + +if [ -z "$2" ]; then + condaenv="nexus-xd-python-modules" +else + condaenv=$2 +fi + +pushd $homedir + +pushd nexus +git pull https://github.com/dataplumber/nexus.git +popd + +source activate $condaenv + +# Install spring-xd python module +pushd nexus/nexus-ingest/spring-xd-python +python setup.py install --force +popd + +# Install protobuf generated artifacts +pushd nexus/nexus-ingest/nexus-messages +./gradlew clean build writeNewPom + +pomfile=`find build/poms/*.xml` +jarfile=`find build/libs/*.jar` +mvn install:install-file -DpomFile=$pomfile -Dfile=$jarfile + +pushd build/python/nexusproto +python setup.py install --force +popd +popd + +# Install ingestion modules +pushd nexus/nexus-ingest/nexus-xd-python-modules +python setup.py install --force +popd + +# Install shared Groovy scripts +pushd nexus/nexus-ingest/groovy-scripts +rm $homedir/xd-nexus-shared/*.groovy +cp *.groovy $homedir/xd-nexus-shared +popd + +# Delete all streams in Spring XD so we can update the custom modules +echo stream all destroy --force > /tmp/xdcommand +$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand + +# Build and upload dataset-tiler +pushd nexus/nexus-ingest/dataset-tiler +./gradlew clean build +jarfile=`find build/libs/*.jar` +touch /tmp/moduleupload +echo module upload --type processor --name dataset-tiler --file $jarfile --force > /tmp/xdcommand +$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand +popd + +# Build and upload tcp-shell +pushd nexus/nexus-ingest/tcp-shell +./gradlew clean build +jarfile=`find build/libs/*.jar` +touch /tmp/moduleupload +echo module upload --type processor --name tcpshell --file $jarfile --force > /tmp/xdcommand +$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand +popd + +# Build and upload nexus-sink +pushd nexus/nexus-ingest/nexus-sink +./gradlew clean build +jarfile=`find build/libs/*.jar` +touch /tmp/moduleupload +echo module upload --type sink --name nexus --file $jarfile --force > /tmp/xdcommand +$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand +popd + +# Create the streams +$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile $scriptdir/stream-definitions + +popd \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/xd-singlenode-logback.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/xd-singlenode-logback.groovy b/nexus-ingest/developer-box/xd-singlenode-logback.groovy new file mode 100644 index 0000000..5aec45b --- /dev/null +++ b/nexus-ingest/developer-box/xd-singlenode-logback.groovy @@ -0,0 +1,91 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +import org.springframework.xd.dirt.util.logging.CustomLoggerConverter +import org.springframework.xd.dirt.util.logging.VersionPatternConverter +import ch.qos.logback.classic.encoder.PatternLayoutEncoder +import ch.qos.logback.core.rolling.RollingFileAppender + +// We highly recommended that you always add a status listener just +// after the last import statement and before all other statements +// NOTE - this includes logging configuration in the log and stacktraces in the event of errors +// statusListener(OnConsoleStatusListener) + +// Emulates Log4j formatting +conversionRule("category", CustomLoggerConverter) + +//XD Version +conversionRule("version", VersionPatternConverter) + +def ISO8601 = "yyyy-MM-dd'T'HH:mm:ssZ" +def datePattern = ISO8601 + +appender("STDOUT", ConsoleAppender) { + encoder(PatternLayoutEncoder) { + pattern = "%d{${datePattern}} %version %level{5} %thread %category{2} - %msg%n" + } +} + +def logfileNameBase = "${System.getProperty('xd.home')}/logs/singlenode-${System.getProperty('PID')}" + +appender("FILE", RollingFileAppender) { + file = "${logfileNameBase}.log" + append = false + rollingPolicy(TimeBasedRollingPolicy) { + fileNamePattern = "${logfileNameBase}-%d{yyyy-MM-dd}.%i.log" + timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) { + maxFileSize = "100KB" + } + } + + encoder(PatternLayoutEncoder) { + pattern = "%d{${datePattern}} %version %level{5} %thread %category{2} - %msg%n" + } +} + +root(WARN, ["STDOUT", "FILE"]) + +logger("org.nasa", INFO) +logger("org.springframework.xd", WARN) +logger("org.springframework.xd.dirt.server", INFO) +logger("org.springframework.xd.dirt.util.XdConfigLoggingInitializer", INFO) +logger("xd.sink", INFO) +logger("org.springframework.xd.sqoop", INFO) +// This is for the throughput-sampler sink module +logger("org.springframework.xd.integration.throughput", INFO) + +logger("org.springframework", WARN) +logger("org.springframework.boot", WARN) +logger("org.springframework.integration", WARN) +logger("org.springframework.retry", WARN) +logger("org.springframework.amqp", WARN) + +// Below this line are specific settings for things that are too noisy +logger("org.springframework.beans.factory.config", ERROR) +logger("org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer", ERROR) + +// This prevents the WARN level InstanceNotFoundException: org.apache.ZooKeeperService:name0=StandaloneServer_port-1 +logger("org.apache.zookeeper.jmx.MBeanRegistry", ERROR) + + +// This prevents the WARN level about a non-static, @Bean method in Spring Batch that is irrelevant +logger("org.springframework.context.annotation.ConfigurationClassEnhancer", ERROR) + +// This prevents the "Error:KeeperErrorCode = NodeExists" INFO messages +// logged by ZooKeeper when a parent node does not exist while +// invoking Curator's creatingParentsIfNeeded node builder. +logger("org.apache.zookeeper.server.PrepRequestProcessor", WARN) + + +// This prevents boot LoggingApplicationListener logger's misleading warning message +logger("org.springframework.boot.logging.LoggingApplicationListener", ERROR) + + + +// This prevents Hadoop configuration warnings +logger("org.apache.hadoop.conf.Configuration", ERROR) + +// Suppress json-path warning until SI 4.2 is released +logger("org.springframework.integration.config.IntegrationRegistrar", ERROR) + http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/.gitignore ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/.gitignore b/nexus-ingest/groovy-scripts/.gitignore new file mode 100644 index 0000000..21e72df --- /dev/null +++ b/nexus-ingest/groovy-scripts/.gitignore @@ -0,0 +1,4 @@ +*.iml + +.idea +/target/ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/README.md ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/README.md b/nexus-ingest/groovy-scripts/README.md new file mode 100644 index 0000000..92e66ba --- /dev/null +++ b/nexus-ingest/groovy-scripts/README.md @@ -0,0 +1,3 @@ +# groovy-scripts + +Various groovy scripts that can be used as part of an [XD Stream](http://docs.spring.io/spring-xd/docs/current/reference/html/#script). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy b/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy new file mode 100644 index 0000000..daf533a --- /dev/null +++ b/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy @@ -0,0 +1,48 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +/** + * Created by greguska on 3/29/16. + */ + +@Grapes([ + @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE') +]) + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent + +if (regex == null ){ + throw new RuntimeException("This script requires a regex to use for matching against the granulename.") +} +if (!(payload instanceof byte[])){ + throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}") +} + +def pattern = ~"$regex" +def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload) + +def tileSummary = tileBuilder.summaryBuilder + +def granulename = tileSummary.granule + +def matches = (granulename =~ pattern) + +if (!matches.hasGroup()){ + throw new RuntimeException("regex did not return any groups.") +} +if (1 != matches.size()){ + throw new RuntimeException("regex did not return *one* group.") +} +if (2 != matches[0].size()){ + throw new RuntimeException("group does not contain match.") +} + +def dayOfYear = matches[0][1] + +tileSummary.addGlobalAttributes(NexusContent.Attribute.newBuilder() + .setName("day_of_year_i") + .addValues(dayOfYear.toString()) + .build()) + +return tileBuilder.build().toByteArray() http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy b/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy new file mode 100644 index 0000000..1591018 --- /dev/null +++ b/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy @@ -0,0 +1,63 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +/** + * Created by greguska on 3/29/16. + */ + +@Grapes([ + @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE') +]) + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent + + +if (regex == null ){ + throw new RuntimeException("This script requires a regex to use for matching against the granulename.") +} +if (dateformat == null ){ + throw new RuntimeException("This script requires a date format to parse the date extracted using the regex.") +} +if (!(payload instanceof byte[])){ + throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}") +} + +def pattern = ~"$regex" +def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload) + +switch(tileBuilder.tile.tileTypeCase){ + case NexusContent.TileData.TileTypeCase.GRID_TILE: + def gridtilebuilder = tileBuilder.tileBuilder.gridTileBuilder + def tilesummary = tileBuilder.summaryBuilder + + def granulename = tilesummary.granule + + def matches = (granulename =~ pattern) + + if (!matches.hasGroup()){ + throw new RuntimeException("regex did not return any groups.") + } + if (1 != matches.size()){ + throw new RuntimeException("regex did not return *one* group.") + } + if (2 != matches[0].size()){ + throw new RuntimeException("group does not contain match.") + } + + def datestring = matches[0][1] + + def time = Date.parse(dateformat, datestring.toString(), TimeZone.getTimeZone("UTC")) + + + def secondsSinceEpoch = (time.time / 1000).toLong() + + gridtilebuilder.setTime(secondsSinceEpoch) + tilesummary.statsBuilder.setMinTime(secondsSinceEpoch) + tilesummary.statsBuilder.setMaxTime(secondsSinceEpoch) + break + default: + throw new RuntimeException("Can only handle GridTile at this time.") +} + +return tileBuilder.build().toByteArray() http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy b/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy new file mode 100644 index 0000000..393164c --- /dev/null +++ b/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy @@ -0,0 +1,34 @@ +/***************************************************************************** + * Copyright (c) 2016 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +/** + * Created by Nga Quach on 8/31/16. + */ + +if (timelen == null) { + throw new RuntimeException("This script requires the length of the time array.") +} +def time = 'time' +if (binding.variables.get("timevar") != null) { + time = timevar +} + +def specsIn = payload +if (payload instanceof String) { + specsIn = [payload] +} + +def length = timelen.toInteger() +def specsOut = [] +for (i = 0; i < length; i++) { + specsOut.addAll(specsIn.collect{ spec -> + "$time:$i:${i+1},$spec" + }) +} + +def sectionSpec = specsOut.join(';') +sectionSpec <<= ';file://' +sectionSpec <<= headers.absolutefilepath + +return sectionSpec.toString() http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/dev-install.sh ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/dev-install.sh b/nexus-ingest/groovy-scripts/dev-install.sh new file mode 100755 index 0000000..44f1bda --- /dev/null +++ b/nexus-ingest/groovy-scripts/dev-install.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +scriptdir=`dirname $0` +sharedscripts=/home/vagrant/xd-nexus-shared + +rm $sharedscripts/*.groovy +cp $scriptdir/*.groovy $sharedscripts \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/generate-tile-id.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/generate-tile-id.groovy b/nexus-ingest/groovy-scripts/generate-tile-id.groovy new file mode 100644 index 0000000..7c4d7de --- /dev/null +++ b/nexus-ingest/groovy-scripts/generate-tile-id.groovy @@ -0,0 +1,37 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +/** + * Created by greguska on 3/29/16. + */ + +@Grapes([ + @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE') +]) + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent + +if (!(payload instanceof byte[])){ + throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}") +} +thesalt = binding.variables.get("salt") + + +def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload) +def summaryBuilder = tileBuilder.getSummaryBuilder() + +def granule = summaryBuilder.hasGranule()?summaryBuilder.granule:"${headers['absolutefilepath']}".split(File.separator)[-1] +def originalSpec = summaryBuilder.hasSectionSpec()?summaryBuilder.sectionSpec:"${headers['spec']}" + + +def tileId = UUID.nameUUIDFromBytes("${granule[0..-4]}$originalSpec${thesalt==null?'':thesalt}".toString().bytes).toString() + +summaryBuilder.setGranule(granule) +summaryBuilder.setTileId(tileId) +summaryBuilder.setSectionSpec(originalSpec) + +tileBuilder.getTileBuilder().setTileId(tileId) + +return tileBuilder.build().toByteArray() + http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/nexustile-to-string.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/nexustile-to-string.groovy b/nexus-ingest/groovy-scripts/nexustile-to-string.groovy new file mode 100644 index 0000000..39030e7 --- /dev/null +++ b/nexus-ingest/groovy-scripts/nexustile-to-string.groovy @@ -0,0 +1,23 @@ +/***************************************************************************** + * Copyright (c) 2016 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +/** + * Created by greguska on 7/18/16. + */ + + +@Grapes([ + @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE') +]) + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent + + +includeData = Boolean.parseBoolean(binding.variables.get("includeData")) + +if(includeData){ + NexusContent.NexusTile.newBuilder().mergeFrom(payload).toString() +}else { + NexusContent.NexusTile.newBuilder().mergeFrom(payload).summary.toString() +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/set-dataset-name.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/set-dataset-name.groovy b/nexus-ingest/groovy-scripts/set-dataset-name.groovy new file mode 100644 index 0000000..ce2f6a2 --- /dev/null +++ b/nexus-ingest/groovy-scripts/set-dataset-name.groovy @@ -0,0 +1,28 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +/** + * Created by greguska on 3/29/16. + */ + +@Grapes([ + @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE') +]) + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent + +if (!(payload instanceof byte[])){ + throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}") +} + +def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload) + + +assert datasetname != null : "This script requires a variable called datasetname." + +def summaryBuilder = tileBuilder.getSummaryBuilder() +summaryBuilder.setDatasetName(datasetname) + +return tileBuilder.build().toByteArray() + http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy b/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy new file mode 100644 index 0000000..0eb2b07 --- /dev/null +++ b/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy @@ -0,0 +1,82 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +/** + * Created by greguska on 3/28/17. + */ +import org.junit.After +import org.junit.Before +import org.junit.Test + +import static groovy.test.GroovyAssert.shouldFail +import static org.junit.Assert.assertEquals + +class TestAddTimeToSpatialSpec { + GroovyShell shell + Binding binding + PrintStream orig + ByteArrayOutputStream out + + File script + + @Before + void setUp() { + orig = System.out + out = new ByteArrayOutputStream() + System.setOut(new PrintStream(out)) + binding = new Binding() + shell = new GroovyShell(binding) + + script = new File('add-time-to-spatial-spec.groovy') + } + + @After + void tearDown() { + System.setOut(orig) + } + + @Test + void testMissingTimeLen() { + def e = shouldFail RuntimeException, { + binding.timelen = null + shell.evaluate(script) + } + assert 'This script requires the length of the time array.' == e.message + } + + @Test + void testStringPayload() { + binding.timelen = 4 + binding.payload = "test:0:1,script:3:4" + binding.headers = ['absolutefilepath': 'afilepath'] + + def expected = [ + "time:0:1,test:0:1,script:3:4", + "time:1:2,test:0:1,script:3:4", + "time:2:3,test:0:1,script:3:4", + "time:3:4,test:0:1,script:3:4" + ].join(';') + ';file://afilepath' + + def result = shell.evaluate(script) + assertEquals expected, result + } + + @Test + void testListPayload() { + binding.timelen = 2 + binding.payload = ["test:0:1,script:3:4", "test:1:2,script:4:5"] + binding.headers = ['absolutefilepath': 'afilepath'] + + def expected = [ + "time:0:1,test:0:1,script:3:4", + "time:0:1,test:1:2,script:4:5", + "time:1:2,test:0:1,script:3:4", + "time:1:2,test:1:2,script:4:5" + ].join(';') + ';file://afilepath' + + def result = shell.evaluate(script) + assertEquals expected, result + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/.gitignore ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/.gitignore b/nexus-ingest/nexus-messages/.gitignore new file mode 100644 index 0000000..ab9b2f9 --- /dev/null +++ b/nexus-ingest/nexus-messages/.gitignore @@ -0,0 +1,35 @@ +.gradle/ +.idea/ +gradlew.bat + +distrobution/ + +gen/ + +build/* +!build/reports + +build/reports/* +!build/reports/license +!build/reports/project + +nexusproto.egg-info/ + +.DS_Store +*.log + +#Idea files +*.iml +*.ipr +*.iws + +*.class + +# Package Files # +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +gradle.properties http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/README.md ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/README.md b/nexus-ingest/nexus-messages/README.md new file mode 100644 index 0000000..34bf7c7 --- /dev/null +++ b/nexus-ingest/nexus-messages/README.md @@ -0,0 +1,24 @@ +# nexus-messages + +This project contains the [protobuf](https://developers.google.com/protocol-buffers/) definition for a NexusTile. By compiling the protobuf specification, both Java and Python objects are generated. + +# Developer Installation + +1. Run `./gradlew clean build install` + +2. cd into `/build/python/nexusproto` + +3. Setup a separate conda env or activate an existing one + + ```` + conda create --name nexus-messages python + source activate nexus-messages + ```` + +4. Install Conda dependencies + + ```` + conda install numpy + ```` + +5. Run `python setup.py install` http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build.gradle ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build.gradle b/nexus-ingest/nexus-messages/build.gradle new file mode 100644 index 0000000..290a338 --- /dev/null +++ b/nexus-ingest/nexus-messages/build.gradle @@ -0,0 +1,195 @@ +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.StandardCopyOption +import java.nio.file.attribute.BasicFileAttributes + +buildscript { + repositories { + if( project.hasProperty('artifactory_contextUrl') ) { + maven { + url "${artifactory_contextUrl}" + credentials { + username = "${artifactory_user}" + password = "${artifactory_password}" + } + } + } + + mavenCentral() + jcenter() + } + dependencies { + //Check for the latest version here: http://plugins.gradle.org/plugin/com.jfrog.artifactory + classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4+" + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.7.7' + } +} + +if( project.hasProperty('artifactory_contextUrl') ) { + allprojects { + apply plugin: "com.jfrog.artifactory" + } + + artifactory { + contextUrl = "${artifactory_contextUrl}" + publish { + repository { + repoKey = "${publish_repoKey}" + username = "${artifactory_user}" + password = "${artifactory_password}" + maven = true + + } + defaults { + // Reference to Gradle publications defined in the build script. + // This is how we tell the Artifactory Plugin which artifacts should be + // published to Artifactory. + publications('mavenJava') + publishArtifacts = true + // Publish generated POM files to Artifactory (true by default) + publishPom = true + } + } + resolve { + repository { + repoKey = "${resolve_repoKey}" + username = "${artifactory_user}" + password = "${artifactory_password}" + maven = true + + } + } + } + + repositories { + maven { + url "$artifactory_contextUrl/$resolve_repoKey" + credentials { + username "${artifactory_user}" + password "${artifactory_password}" + } + } + } + +}else { + repositories { + mavenCentral() + } +} + +apply plugin: 'java' +apply plugin: 'com.google.protobuf' +apply plugin: 'maven' +apply plugin: 'maven-publish' +apply plugin: 'project-report' + + +group = 'org.nasa.jpl.nexus' +version = '1.0.0.RELEASE' + +ext{ + genDirectory = "$projectDir/gen" + distDirectory = "$projectDir/distrobution" + pythonBuildDirPath = "${file(buildDir.path + '/python/nexusproto').path}" +} + +protobuf { + + generatedFilesBaseDir = genDirectory + + // Configure the protoc executable + protoc { + // Download from repositories + artifact = 'com.google.protobuf:protoc:2.6.1' + } + + plugins { + // Define a plugin with name 'grpc' + grpc { + path = 'tools/protoc-gen-grpc-java' + } + } + + generateProtoTasks { + all().each { task -> + task.builtins { + python { + outputSubDir = 'python' + } + } + } + } + +} + +task writeNewPom << { + pom {}.writeTo(file(buildDir.path + "/poms/${project.name}-${project.version}.xml")) +} + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + } + } +} + +assemble.doLast{ + File pythonbuilddir = file(pythonBuildDirPath) + + File pythonsource = file('src/main/python') + + Files.walkFileTree(pythonsource.toPath(), new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + Files.createDirectories(pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(dir))) + return FileVisitResult.CONTINUE + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.copy(file, pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(file)), StandardCopyOption.REPLACE_EXISTING) + return FileVisitResult.CONTINUE + } + }) + + Files.move(file(pythonbuilddir.path + '/nexusproto/setup.py').toPath(), file(pythonbuilddir.path + '/setup.py').toPath(), StandardCopyOption.REPLACE_EXISTING) + + File generatedPython = file("$genDirectory/main/python/NexusContent_pb2.py") + + Files.copy(generatedPython.toPath(), file(pythonbuilddir.path + '/nexusproto/NexusContent_pb2.py').toPath(), StandardCopyOption.REPLACE_EXISTING) +} + + +if( project.hasProperty('python_executable') ) { + task publishPython(type: Exec, group: "publishing", dependsOn: [assemble]) { + workingDir "$pythonBuildDirPath" + commandLine "$python_executable", 'setup.py', 'bdist_wheel', 'upload', '-r', 'deepdata-artifactory-pypi' + } +} + +task tarPython(type: Tar, dependsOn: [assemble]){ + destinationDir = file("distrobution") + archiveName = 'nexusproto.tar.gz' + compression = Compression.GZIP + from(file(buildDir.path + '/python')){ + include '**/*' + } +} + +clean << { + file(genDirectory).deleteDir() + file(distDirectory).deleteDir() +} + +dependencies { + + compile 'com.google.protobuf:protobuf-java:2.6.1' + +} + +task wrapper(type: Wrapper) { + gradleVersion = '2.12' +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html new file mode 100644 index 0000000..9c6d93a --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html @@ -0,0 +1,62 @@ +<html> + <head> + <title>HTML License report</title> + </head> + <style>table { + width: 85%; + border-collapse: collapse; + text-align: center; + } + .dependencies { + text-align: left; + } + tr { + border: 1px solid black; + } + td { + border: 1px solid black; + font-weight: bold; + color: #2E2E2E + } + th { + border: 1px solid black; + } + h3 { + text-align:center; + margin:3px + } + .license { + width:70% + } + + .licenseName { + width:15% + } + </style> + <body> + <table align='center'> + <tr> + <th> + <h3>Dependency</h3> + </th> + <th> + <h3>Jar</h3> + </th> + <th> + <h3>License name</h3> + </th> + <th> + <h3>License text URL</h3> + </th> + </tr> + <tr> + <td class='dependencies'>com.google.protobuf:protobuf-java:2.6.1</td> + <td class='licenseName'>protobuf-java-2.6.1.jar</td> + <td class='licenseName'>New BSD license</td> + <td class='license'> + <a href='http://www.opensource.org/licenses/bsd-license.php'>Show license agreement</a> + </td> + </tr> + </table> + </body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml new file mode 100644 index 0000000..8ed0e97 --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml @@ -0,0 +1,6 @@ +<dependencies> + <dependency name='com.google.protobuf:protobuf-java:2.6.1'> + <file>protobuf-java-2.6.1.jar</file> + <license name='New BSD license' url='http://www.opensource.org/licenses/bsd-license.php' /> + </dependency> +</dependencies> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html new file mode 100644 index 0000000..d9f5e84 --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html @@ -0,0 +1,69 @@ +<html> + <head> + <title>HTML License report</title> + </head> + <style>table { + width: 85%; + border-collapse: collapse; + text-align: center; + } + + .dependencies { + text-align: left; + width:15%; + } + + tr { + border: 1px solid black; + } + + td { + border: 1px solid black; + font-weight: bold; + color: #2E2E2E + } + + th { + border: 1px solid black; + } + + h3 { + text-align:center; + margin:3px + } + + .license { + width:70% + } + + .licenseName { + width:15% + } + </style> + <body> + <table align='center'> + <tr> + <th> + <h3>License</h3> + </th> + <th> + <h3>License text URL</h3> + </th> + <th> + <h3>Dependency</h3> + </th> + </tr> + <tr> + <td class='licenseName'>New BSD license</td> + <td class='license'> + <a href='http://www.opensource.org/licenses/bsd-license.php'>License agreement</a> + </td> + <td class='dependencies'> + <ul> + <li>protobuf-java-2.6.1.jar</li> + </ul> + </td> + </tr> + </table> + </body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml new file mode 100644 index 0000000..66df9d3 --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml @@ -0,0 +1,5 @@ +<licenses> + <license name='New BSD license' url='http://www.opensource.org/licenses/bsd-license.php'> + <dependency>protobuf-java-2.6.1.jar</dependency> + </license> +</licenses> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt b/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt new file mode 100644 index 0000000..6698ae1 --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt @@ -0,0 +1,43 @@ + +------------------------------------------------------------ +Root project +------------------------------------------------------------ + +archives - Configuration for archive artifacts. +No dependencies + +compile - Dependencies for source set 'main'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +compileClasspath - Compile classpath for source set 'main'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +compileOnly - Compile dependencies for source set 'main'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +default - Configuration for default artifacts. +\--- com.google.protobuf:protobuf-java:2.6.1 + +protobuf +No dependencies + +protobufToolsLocator_protoc +\--- com.google.protobuf:protoc:2.6.1 + +runtime - Runtime dependencies for source set 'main'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +testCompile - Dependencies for source set 'test'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +testCompileClasspath - Compile classpath for source set 'test'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +testCompileOnly - Compile dependencies for source set 'test'. +\--- com.google.protobuf:protobuf-java:2.6.1 + +testProtobuf +No dependencies + +testRuntime - Runtime dependencies for source set 'test'. +\--- com.google.protobuf:protobuf-java:2.6.1 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css new file mode 100644 index 0000000..4afa73e --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css @@ -0,0 +1,179 @@ + +body { + margin: 0; + padding: 0; + font-family: sans-serif; + font-size: 12pt; +} + +body, a, a:visited { + color: #303030; +} + +#content { + padding-left: 50px; + padding-right: 50px; + padding-top: 30px; + padding-bottom: 30px; +} + +#content h1 { + font-size: 160%; + margin-bottom: 10px; +} + +#footer { + margin-top: 100px; + font-size: 80%; + white-space: nowrap; +} + +#footer, #footer a { + color: #a0a0a0; +} + +#line-wrapping-toggle { + vertical-align: middle; +} + +#label-for-line-wrapping-toggle { + vertical-align: middle; +} + +ul { + margin-left: 0; +} + +h1, h2, h3 { + white-space: nowrap; +} + +h2 { + font-size: 120%; +} + +ul.tabLinks { + padding-left: 0; + padding-top: 10px; + padding-bottom: 10px; + overflow: auto; + min-width: 800px; + width: auto !important; + width: 800px; +} + +ul.tabLinks li { + float: left; + height: 100%; + list-style: none; + padding-left: 10px; + padding-right: 10px; + padding-top: 5px; + padding-bottom: 5px; + margin-bottom: 0; + -moz-border-radius: 7px; + border-radius: 7px; + margin-right: 25px; + border: solid 1px #d4d4d4; + background-color: #f0f0f0; +} + +ul.tabLinks li:hover { + background-color: #fafafa; +} + +ul.tabLinks li.selected { + background-color: #c5f0f5; + border-color: #c5f0f5; +} + +ul.tabLinks a { + font-size: 120%; + display: block; + outline: none; + text-decoration: none; + margin: 0; + padding: 0; +} + +ul.tabLinks li h2 { + margin: 0; + padding: 0; +} + +div.tab { +} + +div.selected { + display: block; +} + +div.deselected { + display: none; +} + +div.tab table { + min-width: 350px; + width: auto !important; + width: 350px; + border-collapse: collapse; +} + +div.tab th, div.tab table { + border-bottom: solid #d0d0d0 1px; +} + +div.tab th { + text-align: left; + white-space: nowrap; + padding-left: 6em; +} + +div.tab th:first-child { + padding-left: 0; +} + +div.tab td { + white-space: nowrap; + padding-left: 6em; + padding-top: 5px; + padding-bottom: 5px; +} + +div.tab td:first-child { + padding-left: 0; +} + +div.tab td.numeric, div.tab th.numeric { + text-align: right; +} + +span.code { + display: inline-block; + margin-top: 0em; + margin-bottom: 1em; +} + +span.code pre { + font-size: 11pt; + padding-top: 10px; + padding-bottom: 10px; + padding-left: 10px; + padding-right: 10px; + margin: 0; + background-color: #f7f7f7; + border: solid 1px #d0d0d0; + min-width: 700px; + width: auto !important; + width: 700px; +} + +span.wrapped pre { + word-wrap: break-word; + white-space: pre-wrap; + word-break: break-all; +} + +label.hidden { + display: none; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css new file mode 100644 index 0000000..3186aaa --- /dev/null +++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css @@ -0,0 +1,84 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.dependencyTree, #insight { + border: solid 2px #d0d0d0; + border-radius: 10px; + padding: 8px; + background-color: #FFFFEE +} +.alreadyRendered > a, .unresolvable > a { + border-radius: 4px; +} +.alreadyRendered > a { + border: solid 1px #EEEEEE; + background-color: #EEEEEE !IMPORTANT; +} +.hasConflict > a { + font-weight: bold !IMPORTANT; + color: #FAA732 !IMPORTANT; +} +.unresolvable > a { + border: solid 1px #DD514C; + background-color: #DD514C !IMPORTANT; + color: white !IMPORTANT; +} +.leaf a { + font-style: italic !IMPORTANT; +} +.configuration h3 { + cursor: pointer; +} +.configuration h3 ins { + background-image: url(../images/d.png); background-repeat:no-repeat; background-color:white; + background-position: -18px 0; + display: inline-block; + width: 18px; + height: 18px; + position: relative; + top: 4px; +} +.configuration h3.closed ins { + background-position: 0 0; +} +#dependencies { + margin-right: 30px; +} +#insight { + position: fixed; + right: 50px; + top: 50px; + display: none; + width: 70%; + box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.3); +} +#dismissInsight { + float: right; + background:url("../images/d.png") -18px -53px no-repeat !important; + width: 20px; + height: 20px; + cursor: pointer; +} +.insightTree { + height: 450px; + overflow: auto; +} +.insightTree a { + cursor: default; +} +.configurationDescription { + font-size: small; +}
