This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch config_map in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit a53d08e1d24b56bf77ed3a24abb1956c721cb0b1 Author: thomas loubrieu <[email protected]> AuthorDate: Wed Jun 3 19:21:11 2020 -0700 make the config-operator robust to syntactically wrong files, by default synchornization is deactivated --- .../config/{ConfigMap.py => K8ConfigMap.py} | 8 ++++---- sdap_ingest_manager/config/LocalDirConfig.py | 16 ++++++++++++++-- sdap_ingest_manager/config/__init__.py | 6 +++--- sdap_ingest_manager/config_operator.py | 10 ++++++---- setup.py | 10 ++++++---- tests/resources/data/collections.yml | 2 +- tests/resources/data/dataset_config_file_ok.yml | 2 +- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/K8ConfigMap.py similarity index 94% rename from sdap_ingest_manager/config/ConfigMap.py rename to sdap_ingest_manager/config/K8ConfigMap.py index 4143980..156bb19 100644 --- a/sdap_ingest_manager/config/ConfigMap.py +++ b/sdap_ingest_manager/config/K8ConfigMap.py @@ -8,7 +8,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -class ConfigMap: +class K8ConfigMap: def __init__(self, configmap_name, namespace, git_remote_config): self._git_remote_config = git_remote_config self._namespace = namespace @@ -58,10 +58,10 @@ class ConfigMap: finally: return config_keys - def _replace(self): + def _patch(self): try: logger.info(f'replace configMap entry {self._configmap_name}') - api_response = self._api_core_v1_instance.replace_namespaced_config_map( + api_response = self._api_core_v1_instance.patch_namespaced_config_map( name=self._configmap_name, namespace=self._namespace, body=self._create_configmap_object() @@ -87,5 +87,5 @@ class ConfigMap: self._create() except ApiException as e: logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e) - self._replace() + self._patch() diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py index d58f387..515ee22 100644 --- a/sdap_ingest_manager/config/LocalDirConfig.py +++ b/sdap_ingest_manager/config/LocalDirConfig.py @@ -1,6 +1,7 @@ import os import time import logging +import yaml from sdap_ingest_manager.config.exceptions import UnreadableFileException @@ -25,13 +26,24 @@ class LocalDirConfig: return files + def _test_read_yaml(self, file_name): + """ check yaml syntax raiseyaml.parser.ParserError is it doesn't""" + with open(os.path.join(self._local_dir, file_name), 'r') as f: + docs = yaml.load_all(f, Loader=yaml.FullLoader) + for doc in docs: + pass + def get_file_content(self, file_name): logger.info(f'read configuration file {file_name}') try: - with open(os.path.join(self._local_dir, file_name)) as f: - return f.read() + self._test_read_yaml(file_name) + with open(os.path.join(self._local_dir, file_name), 'r') as f: + return f.read() except UnicodeDecodeError as e: raise UnreadableFileException(e) + except yaml.parser.ParserError as e: + raise UnreadableFileException(e) + def _get_latest_update(self): return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir))) diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py index 852920f..918967c 100644 --- a/sdap_ingest_manager/config/__init__.py +++ b/sdap_ingest_manager/config/__init__.py @@ -1,4 +1,4 @@ from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration -from sdap_ingest_manager.config.ConfigMap import ConfigMap -from sdap_ingest_manager.confg.LocalDirConfig import LocalDirConfig -from sdpa_ingest_manager.config.RemoteGitConfig import RemoteGitConfig \ No newline at end of file +from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap +from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig +from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig \ No newline at end of file diff --git a/sdap_ingest_manager/config_operator.py b/sdap_ingest_manager/config_operator.py index fa2771a..97699b9 100644 --- a/sdap_ingest_manager/config_operator.py +++ b/sdap_ingest_manager/config_operator.py @@ -1,6 +1,5 @@ import argparse -from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, ConfigMap - +from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap def main(): @@ -16,6 +15,8 @@ def main(): parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True) parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True) + parser.add_argument("-u", "--updated-continuously", nargs='?', const=True, default=False, + help="k8 configMap is updated as soon as a syntactically correct configuration file is updated") options = parser.parse_args() @@ -24,10 +25,11 @@ def main(): else: config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token) - config_map = ConfigMap(options.config_map, options.namespace, config) + config_map = K8ConfigMap(options.config_map, options.namespace, config) config_map.publish() - config.when_updated(config_map.publish) + if options.updated_continuously: + config.when_updated(config_map.publish) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 75fdd72..7bd9810 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,6 @@ setuptools.setup( long_description_content_type="text/markdown", url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager";, packages=setuptools.find_packages(), - scripts=['bin/run_collections', - 'bin/run_single_collection', - 'bin/run_granules'], classifiers=[ "Programming Language :: Python :: 3", "Operating System :: OS Independent", @@ -53,7 +50,12 @@ setuptools.setup( python_requires='>=3.6', include_package_data=True, data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])], - install_requires=pip_requirements + install_requires=pip_requirements, + entry_points={ + 'config-operator': ['summary=sdap_ingest_manager.config_operator:main'], + 'collection-ingester': ['summary=sdap_ingest_manager.service:main'], + }, + ) post_install_message() diff --git a/tests/resources/data/collections.yml b/tests/resources/data/collections.yml index 18226ba..42d2fbc 100644 --- a/tests/resources/data/collections.yml +++ b/tests/resources/data/collections.yml @@ -1,7 +1,7 @@ avhrr-oi-analysed-sst: path: resources/history_manager/data/avhrr_oi/*.nc variable: analysed_sst - priority: 8 + priority: 9 avhrr-oi-analysed-sst2: path: resources/history_manager/data/avhrr_oi/*.nc diff --git a/tests/resources/data/dataset_config_file_ok.yml b/tests/resources/data/dataset_config_file_ok.yml index a000293..6ff0c47 100644 --- a/tests/resources/data/dataset_config_file_ok.yml +++ b/tests/resources/data/dataset_config_file_ok.yml @@ -7,7 +7,7 @@ ningester: dimensions: - lat - lon - + zobi; --- # Tile processors configuration ningester:
