This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch issue_388_proxy in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f8ad7c77679c15bee852111a4eb99085dc5eab7e Author: Thomas Loubrieu <[email protected]> AuthorDate: Wed Jun 8 15:21:15 2022 +0200 ignore path starting with http, adapt requirements for conda build to work --- collection_manager/collection_manager/entities/Collection.py | 4 ++++ collection_manager/collection_manager/services/CollectionWatcher.py | 5 +++-- config_operator/containers/k8s/config-operator-crd.yml | 3 ++- granule_ingester/conda-requirements.txt | 3 ++- granule_ingester/requirements.txt | 3 ++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/collection_manager/collection_manager/entities/Collection.py b/collection_manager/collection_manager/entities/Collection.py index 0875e7a..e04315e 100644 --- a/collection_manager/collection_manager/entities/Collection.py +++ b/collection_manager/collection_manager/entities/Collection.py @@ -14,9 +14,11 @@ from collection_manager.entities.exceptions import MissingValueCollectionError logger = logging.getLogger(__name__) + class CollectionStorageType(Enum): LOCAL = 1 S3 = 2 + REMOTE = 3 @dataclass(frozen=True) @@ -78,6 +80,8 @@ class Collection: def storage_type(self): if urlparse(self.path).scheme == 's3': return CollectionStorageType.S3 + elif urlparse(self.path).scheme in {'http', 'https'}: + return CollectionStorageType.REMOTE else: return CollectionStorageType.LOCAL diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py index b713f2d..1226351 100644 --- a/collection_manager/collection_manager/services/CollectionWatcher.py +++ b/collection_manager/collection_manager/services/CollectionWatcher.py @@ -84,8 +84,9 @@ class CollectionWatcher: for collection_dict in collections_yaml['collections']: try: collection = Collection.from_dict(collection_dict) - self._validate_collection(collection) - self._collections_by_dir[collection.directory()].add(collection) + if collection.storage_type() != CollectionStorageType.REMOTE: + self._validate_collection(collection) + self._collections_by_dir[collection.directory()].add(collection) except MissingValueCollectionError as e: logger.error(f"A collection is missing '{e.missing_value}'. Ignoring this collection for now.") except RelativePathCollectionError as e: diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml index 2159c00..c02460b 100644 --- a/config_operator/containers/k8s/config-operator-crd.yml +++ b/config_operator/containers/k8s/config-operator-crd.yml @@ -1,5 +1,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition + metadata: name: gitbasedconfigs.sdap.apache.org spec: @@ -76,4 +77,4 @@ spec: containers: - image: tloubrieu/config-operator:latest name: git-repo-config-operator - imagePullPolicy: IfNotPresent \ No newline at end of file + imagePullPolicy: IfNotPresent diff --git a/granule_ingester/conda-requirements.txt b/granule_ingester/conda-requirements.txt index 810e278..a1e4206 100644 --- a/granule_ingester/conda-requirements.txt +++ b/granule_ingester/conda-requirements.txt @@ -5,6 +5,7 @@ pandas==1.0.4 pytz==2019.3 xarray pyyaml==5.3.1 -requests==2.23.0 aiohttp==3.6.2 tenacity +requests==2.27.1 + diff --git a/granule_ingester/requirements.txt b/granule_ingester/requirements.txt index 92f31f3..daf53e9 100644 --- a/granule_ingester/requirements.txt +++ b/granule_ingester/requirements.txt @@ -1,8 +1,9 @@ cassandra-driver==3.23.0 aiomultiprocess==0.7.0 -aioboto3==8.0.5 +aioboto3==8.3.0 tblib==1.6.0 pysolr==3.9.0 kazoo==2.8.0 aio-pika==6.7.1 elasticsearch[async] +urllib3==1.26.2
