This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch config_map in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit b8281218bcbb67df6429eb19766aa023a14158dc Author: thomas loubrieu <[email protected]> AuthorDate: Thu Jun 4 16:18:32 2020 -0700 start to implement config-operator containers --- .github/workflows/github-dev-release.yml | 6 +++ README.md | 56 ++++++++-------------- .../docker/{ => collection-ingester}/Dockerfile | 0 containers/docker/config-operator/Dockerfile | 10 ++++ containers/kubernetes/config-operator.yml | 39 +++++++++++++++ sdap_ingest_manager/__init__.py | 1 + sdap_ingest_manager/config/K8ConfigMap.py | 2 + sdap_ingest_manager/config/LocalDirConfig.py | 2 +- setup.py | 29 ++--------- tests/resources/data/dataset_config_file_ok.yml | 1 - 10 files changed, 85 insertions(+), 61 deletions(-) diff --git a/.github/workflows/github-dev-release.yml b/.github/workflows/github-dev-release.yml index ae6e030..9f9b555 100644 --- a/.github/workflows/github-dev-release.yml +++ b/.github/workflows/github-dev-release.yml @@ -62,5 +62,11 @@ jobs: run: | pip install pds-github-util python-snapshot-release --token ${{ secrets.GITHUB_TOKEN }} + - name: Publish the Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + user: ${{ secrets.pypi_username }} + password: ${{ secrets.pypi_password }} + repository_url: https://test.pypi.org/legacy/ diff --git a/README.md b/README.md index 1dbdf70..d653212 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,27 @@ Deploy a local rabbitmq service, for example with docker. docker run -d --hostname localhost -p 5672:5672 --name rabbitmq rabbitmq:3 + ### Launch the service - python sdap_ingest_manager/service.py --local-ingestion-orders=tests/resources/data/collections.yml --history-path=/tmp +#### The config operator: + +This component helps to import a configuration directory on local file system or on a git repository as a configMap in kubernetes. +This makes the configuration easily accessible to all the nodes of the cluster whereas the configuration stays in a single place. +The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations. + + config-operator -h + config-operator -l tests/resources/data -n sdap -cm collection-ingester-config + config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config + +#### The collection ingestion service + +The service reads the collection configuration and submit granule ingestion messages to the message broker (rabbitmq). +For each collection, 2 ingestion priority levels are proposed: the nominal priority, the priority for forward processing (newer files), usually higher. +An history of the ingested granules is managed so that the ingestion can stop and re-start anytime. + + collection-ingester -h + collection-ingester --local-ingestion-orders=tests/resources/data/collections.yml --history-path=/tmp ### Test and create the package @@ -149,47 +167,15 @@ The release will be automatically pushed to pypi though github action. ## Docker -(development version) - - cd containers/docker - docker build --no-cache --tag tloubrieu/sdap-ingest-manager:latest . - docker run -it --name sdap-ingest-manager -v sdap_ingest_config:/usr/local/.sdap_ingest_manager tloubrieu/sdap-ingest-manager:latest - docker volume inspect sdap_ingest_config - -You can see the configuration files in the directory of the named volume (for example /var/lib/docker/volumes/sdap_ingest_config/_data). - -Note on macos, to access this directory, you need to go inside the Virtual Machine which runs docker service. To update the configuration on macos: - - docker run --rm -it -v /:/vm-root alpine:edge /bin/bash - cd /vm-root/var/lib/docker/volumes/sdap_ingest_config/_data - cp sdap_ingest_manager.ini.default sdap_ingest_manager.ini - vi sdap_ingest_manager.ini - + docker build . -f containers/docker/config-operator/Dockerfile --no-cache --tag tloubrieu/sdap-ingest-manager:latest + To publish the docker image on dockerhub do (step necessary for kubernetes deployment): docker login docker push tloubrieu/sdap-ingest-manager:latest ## Kubernetes - -### Create the configMap for your deployment - -Prepare a configMap from existing native config files: - - kubectl create configmap collection-ingester-config --from-file=venv/.sdap_ingest_manager -n sdap - -#### Optionally you can update the configMap manually if the one you started from is not what you needed: - kubectl get configmap collection-ingester-config -o yaml -n sdap > containers/kubernetes/sdap_ingester_config.yml - -Manually edit the yml file to only keep the configuration which is specific to the deployment (if different from the current one) - -Replace the configmap: - - kubectl delete configmap collection-ingester-config -n sdap - kubectl apply -f containers/kubernetes/sdap_ingester_config.yml -n sdap - - ### Launch the service kubectl apply -f containers/kubernetes/job.yml -n sdap diff --git a/containers/docker/Dockerfile b/containers/docker/collection-ingester/Dockerfile similarity index 100% rename from containers/docker/Dockerfile rename to containers/docker/collection-ingester/Dockerfile diff --git a/containers/docker/config-operator/Dockerfile b/containers/docker/config-operator/Dockerfile new file mode 100644 index 0000000..81918b8 --- /dev/null +++ b/containers/docker/config-operator/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3 + +COPY /sdap_ingest_manager /sdap_ingest_manager/sdap_ingest_manager +COPY /setup.py /sdap_ingest_manager/setup.py +COPY /requirements.txt /sdap_ingest_manager/requirements.txt +COPY /README.md /sdap_ingest_manager/README.md + +RUN cd /sdap_ingest_manager && pip install . + +CMD bash diff --git a/containers/kubernetes/config-operator.yml b/containers/kubernetes/config-operator.yml new file mode 100644 index 0000000..1d8bc16 --- /dev/null +++ b/containers/kubernetes/config-operator.yml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: collection-ingester +spec: + template: + spec: + containers: + - name: collections-ingester + image: tloubrieu/sdap-ingest-manager:latest + imagePullPolicy: IfNotPresent + command: ["run_collections", "--config=/opt/sdap_ingester_config/"] + volumeMounts: + - name: config-vol + mountPath: /opt/sdap_ingester_config/ + - name: data-volume-for-collection-ingester + mountPath: /data + readOnly: true + volumes: + - name: config-vol + configMap: + name: collection-ingester-config + - name: data-volume-for-collection-ingester + #hostPath: + # path: /Users/loubrieu/PycharmProjects/sdap_ingest_manager/sdap_ingest_manager/ingestion_order_executor/history_manager/data + # type: Directory + persistentVolumeClaim: + claimName: data-volume-claim + + restartPolicy: Never + backoffLimit: 4 + +--- + + + + + + diff --git a/sdap_ingest_manager/__init__.py b/sdap_ingest_manager/__init__.py index e69de29..51d7666 100644 --- a/sdap_ingest_manager/__init__.py +++ b/sdap_ingest_manager/__init__.py @@ -0,0 +1 @@ +__version__='1.0.0.dev0' \ No newline at end of file diff --git a/sdap_ingest_manager/config/K8ConfigMap.py b/sdap_ingest_manager/config/K8ConfigMap.py index 156bb19..e9afe55 100644 --- a/sdap_ingest_manager/config/K8ConfigMap.py +++ b/sdap_ingest_manager/config/K8ConfigMap.py @@ -59,6 +59,8 @@ class K8ConfigMap: return config_keys def _patch(self): + """ replaces files available in the config but does not delete + what is not available (e.g. which has not been parsed)""" try: logger.info(f'replace configMap entry {self._configmap_name}') api_response = self._api_core_v1_instance.patch_namespaced_config_map( diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py index 515ee22..6d8a7cf 100644 --- a/sdap_ingest_manager/config/LocalDirConfig.py +++ b/sdap_ingest_manager/config/LocalDirConfig.py @@ -27,7 +27,7 @@ class LocalDirConfig: return files def _test_read_yaml(self, file_name): - """ check yaml syntax raiseyaml.parser.ParserError is it doesn't""" + """ check yaml syntax raise yaml.parser.ParserError is it doesn't""" with open(os.path.join(self._local_dir, file_name), 'r') as f: docs = yaml.load_all(f, Loader=yaml.FullLoader) for doc in docs: diff --git a/setup.py b/setup.py index 7bd9810..f85d6a7 100644 --- a/setup.py +++ b/setup.py @@ -6,22 +6,6 @@ import re PACKAGE_NAME = "sdap_ingest_manager" - -def post_install_message(): - try: - from tabulate import tabulate - except ImportError: - subprocess.call([sys.executable, "-m", "pip", "install", 'tabulate']) - finally: - from tabulate import tabulate - - path_to_configuration_files = os.path.join(sys.prefix, f".{PACKAGE_NAME}") - message = f"Now, create configuration files in \n" \ - f"***{path_to_configuration_files}*** \n" \ - f" Use templates and examples provided there" - print(tabulate([[message]])) - - with open("./sdap_ingest_manager/__init__.py") as fi: result = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fi.read()) version = result.group(1) @@ -29,8 +13,8 @@ version = result.group(1) with open("README.md", "r") as fh: long_description = fh.read() -with open('requirements.txt') as f: - pip_requirements = f.readlines() +with open('requirements.txt', 'r') as f: + requirements = f.readlines() setuptools.setup( name=PACKAGE_NAME, @@ -50,12 +34,9 @@ setuptools.setup( python_requires='>=3.6', include_package_data=True, data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])], - install_requires=pip_requirements, + install_requires=requirements, entry_points={ - 'config-operator': ['summary=sdap_ingest_manager.config_operator:main'], - 'collection-ingester': ['summary=sdap_ingest_manager.service:main'], + 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main', + 'collection-ingester=sdap_ingest_manager.service:main'] }, - ) - -post_install_message() diff --git a/tests/resources/data/dataset_config_file_ok.yml b/tests/resources/data/dataset_config_file_ok.yml index 6ff0c47..66bb883 100644 --- a/tests/resources/data/dataset_config_file_ok.yml +++ b/tests/resources/data/dataset_config_file_ok.yml @@ -7,7 +7,6 @@ ningester: dimensions: - lat - lon - zobi; --- # Tile processors configuration ningester:
