This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 58eccd06d9622c9a6eab9458c2de7abbd66f4a0e Merge: 1fedc94 472ab15 Author: thomas loubrieu <[email protected]> AuthorDate: Fri May 14 13:58:27 2021 -0700 Merge pull request #34 from apache/dev merge dev in master (after 13 months without merge) .gitattributes | 1 - .github/workflows/github-dev-release.yml | 6 + .gitignore | 9 +- .idea/.gitignore | 2 - .idea/inspectionProfiles/profiles_settings.xml | 6 - .idea/misc.xml | 7 - .idea/modules.xml | 8 - .idea/sdap_ingest_manager.iml | 10 - .idea/vcs.xml | 6 - README.md | 199 -------- bin/run_collections | 62 --- bin/run_granules | 5 - bin/run_single_collection | 96 ---- collection_manager/README.md | 121 +++++ collection_manager/collection_manager/__init__.py | 1 + .../collection_manager/entities/Collection.py | 74 +++ .../collection_manager/entities/__init__.py | 2 + .../entities/exceptions/Exceptions.py | 30 ++ .../entities/exceptions/__init__.py | 6 + collection_manager/collection_manager/main.py | 100 ++++ .../services/CollectionProcessor.py | 112 ++++ .../services/CollectionWatcher.py | 222 ++++++++ .../services/MessagePublisher.py | 40 ++ .../collection_manager/services/S3Observer.py | 147 ++++++ .../collection_manager/services}/__init__.py | 6 + .../history_manager/FileIngestionHistory.py | 95 ++-- .../services/history_manager/IngestionHistory.py | 139 +++++ .../history_manager/SolrIngestionHistory.py | 153 ++++++ .../services/history_manager/__init__.py | 4 + collection_manager/docker/Dockerfile | 20 + collection_manager/docker/entrypoint.sh | 12 + .../migration}/__init__.py | 0 .../migration/history_migration.py.bkp | 33 ++ collection_manager/requirements.txt | 9 + collection_manager/setup.py | 35 ++ .../test => collection_manager/tests}/__init__.py | 0 .../tests/entities}/__init__.py | 0 .../tests/entities/test_Collection.py | 177 +++++++ collection_manager/tests/resources/collections.yml | 47 ++ .../tests/resources/collections_alternate.yml | 45 ++ .../tests/resources/collections_bad_schema.yml | 17 + .../tests/resources/collections_bad_syntax.yml | 17 + .../resources}/data/avhrr-oi-analysed-sst.csv | 0 ...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0 ...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | 0 .../tests/services}/__init__.py | 0 .../tests/services/history_manager}/__init__.py | 0 .../history_manager/test_FileIngestionHistory.py | 64 +++ .../history_manager/test_SolrIngestionHistory.py | 17 +- .../tests/services/test_CollectionProcessor.py | 201 ++++++++ .../tests/services/test_CollectionWatcher.py | 255 ++++++++++ .../tests/services/test_S3Observer.py | 8 + .../test => common/common}/__init__.py | 0 common/common/async_test_utils/AsyncTestUtils.py | 28 + common/common/async_test_utils/__init__.py | 1 + common/common/async_utils/AsyncUtils.py | 11 + common/common/async_utils/__init__.py | 1 + common/setup.py | 21 + config_operator/README.md | 68 +++ config_operator/config_operator/__init__.py | 1 + .../config_source/LocalDirConfig.py | 80 +++ .../config_source/RemoteGitConfig.py | 82 +++ .../config_operator/config_source/__init__.py | 2 + .../config_operator/config_source/exceptions.py | 4 + .../config_operator/k8s/K8sConfigMap.py | 111 ++++ config_operator/config_operator/k8s/__init__.py | 1 + config_operator/config_operator/main.py | 69 +++ config_operator/containers/docker/Dockerfile | 11 + .../containers/k8s/config-operator-crd.yml | 79 +++ config_operator/containers/k8s/git-repo-test.yml | 9 + config_operator/requirements.txt | 5 + config_operator/setup.py | 38 ++ .../test => config_operator/tests}/__init__.py | 0 .../tests/config_source}/__init__.py | 0 .../tests/config_source/test_LocalDirConfig.py | 72 +++ .../tests/config_source/test_RemoteGitConfig.py | 49 ++ .../test => config_operator/tests/k8s}/__init__.py | 0 config_operator/tests/k8s/test_K8sConfigMap.py | 74 +++ config_operator/tests/resources/collections.yml | 18 + .../resources/localDirBadTest/collections.yml | 2 + .../tests/resources/localDirTest/.hidden_file.txt | 1 + .../tests/resources/localDirTest/README.md | 1 + .../tests/resources/localDirTest/collections.yml | 1 + containers/docker/Dockerfile | 11 - containers/kubernetes/job.yml | 25 - containers/kubernetes/sdap_ingester_config.yml | 38 -- granule_ingester/README.md | 35 ++ granule_ingester/conda-requirements.txt | 10 + granule_ingester/docker/Dockerfile | 25 + granule_ingester/docker/entrypoint.sh | 14 + granule_ingester/docker/install_nexusproto.sh | 20 + granule_ingester/granule_ingester/README.md | 11 + .../granule_ingester}/__init__.py | 0 .../granule_ingester/consumer/MessageConsumer.py | 113 +++++ .../granule_ingester/consumer/__init__.py | 1 + .../granule_ingester/exceptions/Exceptions.py | 46 ++ .../granule_ingester/exceptions/__init__.py | 8 + .../granule_loaders/GranuleLoader.py | 78 +++ .../granule_ingester/granule_loaders/__init__.py | 1 + .../granule_ingester/healthcheck/HealthCheck.py | 8 + .../granule_ingester/healthcheck/__init__.py | 1 + granule_ingester/granule_ingester/main.py | 152 ++++++ .../granule_ingester/pipeline/Modules.py | 27 + .../granule_ingester/pipeline/Pipeline.py | 209 ++++++++ .../granule_ingester/pipeline/__init__.py | 2 + .../granule_ingester/processors/EmptyTileFilter.py | 42 ++ .../processors/ForceAscendingLatitude.py | 48 ++ .../granule_ingester/processors/GenerateTileId.py | 27 +- .../processors/Subtract180FromLongitude.py | 38 ++ .../granule_ingester/processors/TileProcessor.py | 41 ++ .../processors/TileSummarizingProcessor.py | 102 ++++ .../granule_ingester/processors/__init__.py | 7 + .../granule_ingester/processors/kelvintocelsius.py | 43 ++ .../reading_processors/EccoReadingProcessor.py | 64 +++ .../reading_processors/GridReadingProcessor.py | 53 ++ .../reading_processors/SwathReadingProcessor.py | 47 ++ .../reading_processors/TileReadingProcessor.py | 79 +++ .../TimeSeriesReadingProcessor.py | 83 +++ .../processors/reading_processors/__init__.py | 5 + .../slicers/SliceFileByDimension.py | 55 ++ .../slicers/SliceFileByStepSize.py | 55 ++ .../slicers/SliceFileByTilesDesired.py | 68 +++ .../granule_ingester/slicers/TileSlicer.py | 56 ++ .../granule_ingester/slicers/__init__.py | 2 + .../granule_ingester/writers/CassandraStore.py | 97 ++++ .../granule_ingester/writers/DataStore.py | 13 + .../granule_ingester/writers/MetadataStore.py | 11 + .../granule_ingester/writers/SolrStore.py | 170 +++++++ .../granule_ingester/writers/__init__.py | 4 + granule_ingester/requirements.txt | 7 + granule_ingester/setup.py | 34 ++ .../test => granule_ingester/tests}/__init__.py | 0 .../tests/config_files/analysed_sst.yml | 16 + .../config_files/ingestion_config_testfile.yaml | 17 + ...4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc | Bin 0 -> 1057327 bytes .../granules/HLS.S30.T11SPC.2020001.v1.4.hdf.nc | Bin 0 -> 167801 bytes granule_ingester/tests/granules/OBP_2017_01.nc | Bin 0 -> 2110135 bytes granule_ingester/tests/granules/OBP_native_grid.nc | Bin 0 -> 1285094 bytes .../SMAP_L2B_SSS_04892_20160101T005507_R13080.h5 | Bin 0 -> 18672352 bytes granule_ingester/tests/granules/THETA_199201.nc | Bin 0 -> 4255957 bytes granule_ingester/tests/granules/empty_mur.nc4 | Bin 0 -> 60937 bytes .../tests/granules/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes .../tests/granules/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes granule_ingester/tests/granules/not_empty_ccmp.nc | Bin 0 -> 206870 bytes granule_ingester/tests/granules/not_empty_mur.nc4 | Bin 0 -> 60907 bytes granule_ingester/tests/granules/not_empty_smap.h5 | Bin 0 -> 3000192 bytes granule_ingester/tests/granules/not_empty_wswm.nc | Bin 0 -> 1041568 bytes .../tests/pipeline}/__init__.py | 0 granule_ingester/tests/pipeline/test_Pipeline.py | 106 ++++ .../tests/processors}/__init__.py | 0 .../processors/test_ForceAscendingLatitude.py | 58 +++ .../tests/processors/test_GenerateTileId.py | 22 + .../tests/reading_processors}/__init__.py | 0 .../test_EccoReadingProcessor.py | 64 +++ .../test_GridReadingProcessor.py | 265 ++++++++++ .../test_SwathReadingProcessor.py | 74 +++ .../test_TileReadingProcessor.py | 29 ++ .../test_TileSummarizingProcessor.py | 44 ++ .../test_TimeSeriesReadingProcessor.py | 86 ++++ .../tests/slicers}/__init__.py | 0 .../tests/slicers/test_SliceFileByDimension.py | 122 +++++ .../tests/slicers/test_SliceFileByStepSize.py | 105 ++++ .../tests/slicers/test_SliceFileByTilesDesired.py | 88 ++++ granule_ingester/tests/slicers/test_TileSlicer.py | 68 +++ .../tests/writers}/__init__.py | 0 granule_ingester/tests/writers/test_SolrStore.py | 69 +++ requirements.txt | 6 - sdap_ingest_manager/__init__.py | 1 - .../collections_ingester/MANIFEST.in | 4 - .../collections_ingester/collection_ingestion.py | 220 -------- .../google_spreadsheet_collection_config.py | 69 --- .../collections_ingester/nfs_mount_parse.py | 55 -- .../resources/config/collections.yml.example | 9 - .../resources/config/credentials.json.template | 10 - .../resources/config/sdap_ingest_manager.ini | 15 - .../config/sdap_ingest_manager.ini.default | 50 -- .../resources/dataset_config_template.yml | 45 -- .../collections_ingester/test/data/collections.yml | 9 - .../test/data/dataset_config_file_ok.yml | 45 -- .../test/unit/collection_ingestion_test.py | 151 ------ .../test/unit/nfs_mount_parse_test.py | 29 -- .../collections_ingester/test/unit/util_test.py | 17 - .../test/unit/yaml_file_collection_config_test.py | 29 -- .../test/validation/__init__.py | 13 - .../test/validation/validation_test.py | 136 ----- sdap_ingest_manager/collections_ingester/util.py | 26 - .../yaml_file_collection_config.py | 32 -- sdap_ingest_manager/granule_ingester/MANIFEST.in | 1 - sdap_ingest_manager/granule_ingester/Readme.md | 138 ----- sdap_ingest_manager/granule_ingester/__init__.py | 4 - .../resources/connection-config.yml | 14 - .../resources/job-deployment-template.yml | 47 -- sdap_ingest_manager/granule_ingester/runjobs.py | 563 --------------------- sdap_ingest_manager/history_manager/__init__.py | 3 - .../history_manager/datasetingestionhistorysolr.py | 114 ----- .../test/datasetingestionhistoryfile_test.py | 64 --- sdap_ingest_manager/history_manager/util.py | 17 - setup.cfg | 5 - setup.py | 76 --- 199 files changed, 6230 insertions(+), 2572 deletions(-)
