This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch config_map in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 49042bb822c2c2e9ab3eb26dd2d07e4b2992f383 Author: thomas loubrieu <[email protected]> AuthorDate: Thu Jun 4 20:19:42 2020 -0700 move files in specific subdirectory, start k8s deployment, does not work --- README.md | 11 ------- config_operator/README.md | 38 ++++++++++++++++++++++ config_operator/config_operator/__init__.py | 1 + .../config_operator}/config_operator.py | 5 +-- .../config_source}/LocalDirConfig.py | 5 ++- .../config_source}/RemoteGitConfig.py | 27 +++++++-------- .../config_operator/config_source/__init__.py | 2 ++ .../config_operator/config_source}/exceptions.py | 2 +- .../config_operator/k8s/K8sConfigMap.py | 4 +-- config_operator/config_operator/k8s/__init__.py | 1 + config_operator/containers/docker/Dockerfile | 10 ++++++ config_operator/containers/k8s/deployment.yml | 21 ++++++++++++ config_operator/requirements.txt | 3 ++ setup.py => config_operator/setup.py | 16 ++++----- .../config => config_operator/tests}/__init__.py | 0 .../tests}/config/__init__.py | 0 config_operator/tests/config/test_ConfigMap.py | 27 +++++++++++++++ config_operator/tests/resources/collections.yml | 9 +++++ containers/docker/config-operator/Dockerfile | 10 ------ sdap_ingest_manager/config/__init__.py | 3 -- setup.py | 2 +- tests/config/test_ConfigMap.py | 22 ------------- 22 files changed, 141 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index d653212..adb6ddd 100644 --- a/README.md +++ b/README.md @@ -128,17 +128,6 @@ Deploy a local rabbitmq service, for example with docker. ### Launch the service -#### The config operator: - -This component helps to import a configuration directory on local file system or on a git repository as a configMap in kubernetes. -This makes the configuration easily accessible to all the nodes of the cluster whereas the configuration stays in a single place. -The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations. - - config-operator -h - config-operator -l tests/resources/data -n sdap -cm collection-ingester-config - config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config - -#### The collection ingestion service The service reads the collection configuration and submit granule ingestion messages to the message broker (rabbitmq). For each collection, 2 ingestion priority levels are proposed: the nominal priority, the priority for forward processing (newer files), usually higher. diff --git a/config_operator/README.md b/config_operator/README.md new file mode 100644 index 0000000..61889c4 --- /dev/null +++ b/config_operator/README.md @@ -0,0 +1,38 @@ +# Config Operator + +## Purpose + +Component which synchonizes local configuration in a directory, on a file system, or configuration files managed in a git repository with kubernetes configMap. +This helps to make a configuration managed by the operators in a single place (git, host file system) available in the kubernetes cluster. + +For SDAP, it is used to make the configuration of the collections to be ingested available to the ingester service pods. + +# Launch the service + +The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations. + + config-operator -h + config-operator -l tests/resources/data -n sdap -cm collection-ingester-config + config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config + +# Developers + + git clone ... + cd config_operator + pip install -e . + pytest -d + +# Containerizaion + +## Docker + + docker build . -f containers/docker/Dockerfile --no-cache --tag tloubrieu/config-operator:latest + +To publish the docker image on dockerhub do (step necessary for kubernetes deployment): + + docker login + docker push tloubrieu/sdap-ingest-manager:latest + +## Kubernetes + + kubectl apply -f containers/k8s/deployment.yml -n sdap \ No newline at end of file diff --git a/config_operator/config_operator/__init__.py b/config_operator/config_operator/__init__.py new file mode 100644 index 0000000..7d24cef --- /dev/null +++ b/config_operator/config_operator/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0.dev0' diff --git a/sdap_ingest_manager/config_operator.py b/config_operator/config_operator/config_operator.py similarity index 89% rename from sdap_ingest_manager/config_operator.py rename to config_operator/config_operator/config_operator.py index 97699b9..b032384 100644 --- a/sdap_ingest_manager/config_operator.py +++ b/config_operator/config_operator/config_operator.py @@ -1,5 +1,6 @@ import argparse -from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap +from config_operator.config_source import RemoteGitConfig, LocalDirConfig +from config_operator.k8s import K8sConfigMap def main(): @@ -25,7 +26,7 @@ def main(): else: config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token) - config_map = K8ConfigMap(options.config_map, options.namespace, config) + config_map = K8sConfigMap(options.config_map, options.namespace, config) config_map.publish() if options.updated_continuously: diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py similarity index 93% rename from sdap_ingest_manager/config/LocalDirConfig.py rename to config_operator/config_operator/config_source/LocalDirConfig.py index 6d8a7cf..89c0e5a 100644 --- a/sdap_ingest_manager/config/LocalDirConfig.py +++ b/config_operator/config_operator/config_source/LocalDirConfig.py @@ -3,7 +3,8 @@ import time import logging import yaml -from sdap_ingest_manager.config.exceptions import UnreadableFileException + +from config_operator.config_source.exceptions import UnreadableFileException logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -12,7 +13,9 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1 class LocalDirConfig: + def __init__(self, local_dir): + logger.info(f'create config on local dir {local_dir}') self._local_dir = local_dir self._latest_update = self._get_latest_update() diff --git a/sdap_ingest_manager/config/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py similarity index 77% rename from sdap_ingest_manager/config/RemoteGitConfig.py rename to config_operator/config_operator/config_source/RemoteGitConfig.py index a344246..24e614a 100644 --- a/sdap_ingest_manager/config/RemoteGitConfig.py +++ b/config_operator/config_operator/config_source/RemoteGitConfig.py @@ -2,19 +2,19 @@ import logging import os import sys import time -from git import Repo, Remote -from sdap_ingest_manager.config import LocalDirConfig - +from git import Repo +from .LocalDirConfig import LocalDirConfig logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5 + class RemoteGitConfig(LocalDirConfig): def __init__(self, git_url, - git_branch='master', - git_token=None + branch='master', + token=None ): """ @@ -23,18 +23,18 @@ class RemoteGitConfig(LocalDirConfig): :param git_token: """ self._git_url = git_url if git_url.endswith(".git") else git_url + '.git' - self._git_branch = git_branch - self._git_token = git_token + self._git_branch = branch + self._git_token = token local_dir = os.path.join(sys.prefix, 'sdap', 'conf') super().__init__(local_dir) self._repo = None self._init_local_config_repo() - self._latest_commit_key = self._repo.head.commit.hexsha + self._latest_commit_key = self._pull_remote() def _pull_remote(self): o = self._repo.remotes.origin res = o.pull() - return res[0].commit.hexsha # return the latest commit key + return res[0].commit.hexsha # return the latest commit key def _init_local_config_repo(self): self._repo = Repo.init(self._local_dir) @@ -43,6 +43,8 @@ class RemoteGitConfig(LocalDirConfig): self._repo.git.fetch() self._repo.git.checkout(self._git_branch) + + def when_updated(self, callback): while True: @@ -54,9 +56,4 @@ class RemoteGitConfig(LocalDirConfig): self._latest_commit_key = remote_commit_key else: logger.debug("remote git repository has not been updated") - - - - - - + diff --git a/config_operator/config_operator/config_source/__init__.py b/config_operator/config_operator/config_source/__init__.py new file mode 100644 index 0000000..b23e9be --- /dev/null +++ b/config_operator/config_operator/config_source/__init__.py @@ -0,0 +1,2 @@ +from .RemoteGitConfig import RemoteGitConfig +from .LocalDirConfig import LocalDirConfig diff --git a/sdap_ingest_manager/config/exceptions.py b/config_operator/config_operator/config_source/exceptions.py similarity index 83% rename from sdap_ingest_manager/config/exceptions.py rename to config_operator/config_operator/config_source/exceptions.py index c06b881..96e8502 100644 --- a/sdap_ingest_manager/config/exceptions.py +++ b/config_operator/config_operator/config_source/exceptions.py @@ -1,4 +1,4 @@ class UnreadableFileException(Exception): - pass \ No newline at end of file + pass diff --git a/sdap_ingest_manager/config/K8ConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py similarity index 96% rename from sdap_ingest_manager/config/K8ConfigMap.py rename to config_operator/config_operator/k8s/K8sConfigMap.py index e9afe55..b16b58c 100644 --- a/sdap_ingest_manager/config/K8ConfigMap.py +++ b/config_operator/config_operator/k8s/K8sConfigMap.py @@ -2,13 +2,13 @@ import logging from kubernetes import client, config from kubernetes.client.rest import ApiException -from sdap_ingest_manager.config.exceptions import UnreadableFileException +from config_operator.config_source.exceptions import UnreadableFileException logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -class K8ConfigMap: +class K8sConfigMap: def __init__(self, configmap_name, namespace, git_remote_config): self._git_remote_config = git_remote_config self._namespace = namespace diff --git a/config_operator/config_operator/k8s/__init__.py b/config_operator/config_operator/k8s/__init__.py new file mode 100644 index 0000000..2d5a84d --- /dev/null +++ b/config_operator/config_operator/k8s/__init__.py @@ -0,0 +1 @@ +from .K8sConfigMap import K8sConfigMap diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile new file mode 100644 index 0000000..4e82c98 --- /dev/null +++ b/config_operator/containers/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3 + +COPY /config_operator /config_operator/config_operator +COPY /setup.py /config_operator/setup.py +COPY /requirements.txt /config_operator/requirements.txt +COPY /README.md /config_operator/README.md + +RUN cd /config_operator && pip install . + +CMD bash diff --git a/config_operator/containers/k8s/deployment.yml b/config_operator/containers/k8s/deployment.yml new file mode 100644 index 0000000..1501a75 --- /dev/null +++ b/config_operator/containers/k8s/deployment.yml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: config-operator + labels: + app: sdap-config-operator +spec: + replicas: 1 + selector: + matchLabels: + app: sdap-config-operator + template: + metadata: + labels: + app: sdap-config-operator + spec: + containers: + - name: sdap-config-operator + image: tloubrieu/config-operator:latest + imagePullPolicy: IfNotPresent + command: ['config-operator', '--git-url', 'https://github.com/tloubrieu-jpl/sdap-ingester-config' , '--namespace', 'sdap', '--config-map', 'collection-ingester-conf', '-u'] diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt new file mode 100644 index 0000000..4365d3d --- /dev/null +++ b/config_operator/requirements.txt @@ -0,0 +1,3 @@ +GitPython==3.1.2 +kubernetes==11.0 + diff --git a/setup.py b/config_operator/setup.py similarity index 65% copy from setup.py copy to config_operator/setup.py index f85d6a7..2d4c533 100644 --- a/setup.py +++ b/config_operator/setup.py @@ -1,15 +1,13 @@ import setuptools -import os -import subprocess -import sys import re -PACKAGE_NAME = "sdap_ingest_manager" +PACKAGE_NAME = "config_operator" -with open("./sdap_ingest_manager/__init__.py") as fi: +with open(f'./{PACKAGE_NAME}/__init__.py') as fi: result = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fi.read()) version = result.group(1) + with open("README.md", "r") as fh: long_description = fh.read() @@ -21,7 +19,7 @@ setuptools.setup( version=version, author="Apache - SDAP", author_email="[email protected]", - description="a helper to ingest data in sdap", + description="a service to synchronize git or local directory configuration with k8s configMap", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager";, @@ -33,10 +31,8 @@ setuptools.setup( ], python_requires='>=3.6', include_package_data=True, - data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])], install_requires=requirements, entry_points={ - 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main', - 'collection-ingester=sdap_ingest_manager.service:main'] - }, + 'console_scripts': ['config-operator=config_operator.config_operator:main'] + } ) diff --git a/tests/config/__init__.py b/config_operator/tests/__init__.py similarity index 100% copy from tests/config/__init__.py copy to config_operator/tests/__init__.py diff --git a/tests/config/__init__.py b/config_operator/tests/config/__init__.py similarity index 100% rename from tests/config/__init__.py rename to config_operator/tests/config/__init__.py diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/config/test_ConfigMap.py new file mode 100644 index 0000000..4d339e3 --- /dev/null +++ b/config_operator/tests/config/test_ConfigMap.py @@ -0,0 +1,27 @@ +import unittest +import os + +from config_operator.k8s import K8sConfigMap +from config_operator.config_source import RemoteGitConfig, LocalDirConfig + + +class ConfigMapTest(unittest.TestCase): + def test_createconfigmapfromgit(self): + + remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";) + + config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) + config_map.publish() + + def test_createconfigmapfromlocaldir(self): + local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', + 'resources') + remote_git_config = LocalDirConfig(local_dir) + + config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) + config_map.publish() + + +if __name__ == '__main__': + unittest.main() diff --git a/config_operator/tests/resources/collections.yml b/config_operator/tests/resources/collections.yml new file mode 100644 index 0000000..42d2fbc --- /dev/null +++ b/config_operator/tests/resources/collections.yml @@ -0,0 +1,9 @@ +avhrr-oi-analysed-sst: + path: resources/history_manager/data/avhrr_oi/*.nc + variable: analysed_sst + priority: 9 + +avhrr-oi-analysed-sst2: + path: resources/history_manager/data/avhrr_oi/*.nc + variable: analysed_sst + priority: 1 diff --git a/containers/docker/config-operator/Dockerfile b/containers/docker/config-operator/Dockerfile deleted file mode 100644 index 81918b8..0000000 --- a/containers/docker/config-operator/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3 - -COPY /sdap_ingest_manager /sdap_ingest_manager/sdap_ingest_manager -COPY /setup.py /sdap_ingest_manager/setup.py -COPY /requirements.txt /sdap_ingest_manager/requirements.txt -COPY /README.md /sdap_ingest_manager/README.md - -RUN cd /sdap_ingest_manager && pip install . - -CMD bash diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py index 918967c..27a89b8 100644 --- a/sdap_ingest_manager/config/__init__.py +++ b/sdap_ingest_manager/config/__init__.py @@ -1,4 +1 @@ from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration -from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap -from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig -from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig \ No newline at end of file diff --git a/setup.py b/setup.py index f85d6a7..eeed194 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ setuptools.setup( data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])], install_requires=requirements, entry_points={ - 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main', + 'console_scripts': ['config-operator=config_operator.config_operator:main', 'collection-ingester=sdap_ingest_manager.service:main'] }, ) diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py deleted file mode 100644 index 2518b9a..0000000 --- a/tests/config/test_ConfigMap.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -import os - -from flask import Flask -from flask_restplus import Api - -from sdap_ingest_manager.config.ConfigMap import ConfigMap -from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig - - -class ConfigMapTest(unittest.TestCase): - def test_createconfigmap(self): - - remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";) - - config_map = ConfigMap('collection-ingester', 'sdap', remote_git_config) - config_map.publish() - - - -if __name__ == '__main__': - unittest.main()
