This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch config_map in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 5fa6fb4060dc6a9252d2d840ed5f3dbbe91ffe7b Author: thomas loubrieu <[email protected]> AuthorDate: Wed Jun 3 16:54:30 2020 -0700 add config_operator working on local dir or remote git repo --- .gitignore | 4 +- sdap_ingest_manager/config/ConfigMap.py | 69 +++++++++++++++++---------- sdap_ingest_manager/config/LocalDirConfig.py | 49 +++++++++++++++++++ sdap_ingest_manager/config/RemoteGitConfig.py | 62 ++++++++++++++++++++++++ sdap_ingest_manager/config/__init__.py | 3 ++ sdap_ingest_manager/config/exceptions.py | 4 ++ sdap_ingest_manager/config_operator.py | 35 ++++++++++++++ tests/config/test_ConfigMap.py | 16 ++----- tests/resources/data/collections.yml | 4 +- 9 files changed, 206 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index b5cf7f0..0a97257 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ venv __pycache__/ dist/ build/ -*.DS_Store \ No newline at end of file +*.DS_Store +.eggs +temp/ diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/ConfigMap.py index c5eb244..4143980 100644 --- a/sdap_ingest_manager/config/ConfigMap.py +++ b/sdap_ingest_manager/config/ConfigMap.py @@ -2,22 +2,25 @@ import logging from kubernetes import client, config from kubernetes.client.rest import ApiException +from sdap_ingest_manager.config.exceptions import UnreadableFileException + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + class ConfigMap: - def __init__(self, configmap_name, namespace, ingestion_order_store, output_collection='collections.yml'): - self._ingestion_order_store = ingestion_order_store + def __init__(self, configmap_name, namespace, git_remote_config): + self._git_remote_config = git_remote_config self._namespace = namespace self._configmap_name = configmap_name - self._output_collection = output_collection + config.load_kube_config() configuration = client.Configuration() - self._api_instance = client.CoreV1Api(client.ApiClient(configuration)) - + self._api_instance = client.ApiClient(configuration) + self._api_core_v1_instance = client.CoreV1Api(self._api_instance) def __del__(self): - pass + self._api_instance.close() def _create_configmap_object(self): @@ -25,9 +28,14 @@ class ConfigMap: name=self._configmap_name , namespace=self._namespace, ) - - data = {self._output_collection:self._ingestion_order_store.get_content()} + data = {} + for f in self._git_remote_config.get_files(): + try: + data[f] = self._git_remote_config.get_file_content(f) + except UnreadableFileException as e: + logger.error(f'file {f} cannot be read, ignored', e) + configmap = client.V1ConfigMap( api_version="v1", kind="ConfigMap", @@ -37,8 +45,11 @@ class ConfigMap: return configmap def _get_deployed_config(self): + """ + This method does not work in my test, the list of config available is not up to date + """ try: - api_list_response = self._api_instance.list_namespaced_config_map(self._namespace) + api_list_response = self._api_core_v1_instance.list_namespaced_config_map(self._namespace) config_keys = set() for item in api_list_response.items: config_keys = config_keys.union(item.data.keys()) @@ -47,24 +58,34 @@ class ConfigMap: finally: return config_keys - def publish(self): + def _replace(self): try: + logger.info(f'replace configMap entry {self._configmap_name}') + api_response = self._api_core_v1_instance.replace_namespaced_config_map( + name=self._configmap_name, + namespace=self._namespace, + body=self._create_configmap_object() + ) + logger.info(api_response) + except ApiException as e: + raise e - if self._output_collection in self._get_deployed_config(): - logger.info(f'replace configMap entry {self._output_collection}') - api_response = self._api_instance.replace_namespaced_config_map( - name=self._output_collection, - namespace=self._namespace, - body=self._create_configmap_object() - ) - else: - logger.info(f'create configMap entry {self._output_collection}') - api_response = self._api_instance.create_namespaced_config_map( - namespace=self._namespace, - body=self._create_configmap_object() - ) + def _create(self): + try: + logger.info(f'create configMap entry {self._configmap_name}') + api_response = self._api_core_v1_instance.create_namespaced_config_map( + namespace=self._namespace, + body=self._create_configmap_object() + ) logger.info(api_response) except ApiException as e: - logger.error("Exception when calling Kubernetes CoreV1Api %s\n" % e) + raise e + + def publish(self): + try: + self._create() + except ApiException as e: + logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e) + self._replace() diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py new file mode 100644 index 0000000..d58f387 --- /dev/null +++ b/sdap_ingest_manager/config/LocalDirConfig.py @@ -0,0 +1,49 @@ +import os +import time +import logging + +from sdap_ingest_manager.config.exceptions import UnreadableFileException + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1 + + +class LocalDirConfig: + def __init__(self, local_dir): + self._local_dir = local_dir + self._latest_update = self._get_latest_update() + + def get_files(self): + files = [] + for f in os.listdir(self._local_dir): + if os.path.isfile(os.path.join(self._local_dir, f)) \ + and 'README' not in f \ + and not f.startswith('.'): + files.append(f) + + return files + + def get_file_content(self, file_name): + logger.info(f'read configuration file {file_name}') + try: + with open(os.path.join(self._local_dir, file_name)) as f: + return f.read() + except UnicodeDecodeError as e: + raise UnreadableFileException(e) + + def _get_latest_update(self): + return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir))) + + def when_updated(self, callback): + while True: + time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS) + latest_update = self._get_latest_update() + if latest_update > self._latest_update: + logger.info("local config dir has been updated") + callback() + self._latest_update = latest_update + else: + logger.debug("local config dir has not been updated") + diff --git a/sdap_ingest_manager/config/RemoteGitConfig.py b/sdap_ingest_manager/config/RemoteGitConfig.py new file mode 100644 index 0000000..a344246 --- /dev/null +++ b/sdap_ingest_manager/config/RemoteGitConfig.py @@ -0,0 +1,62 @@ +import logging +import os +import sys +import time +from git import Repo, Remote +from sdap_ingest_manager.config import LocalDirConfig + + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5 + +class RemoteGitConfig(LocalDirConfig): + def __init__(self, git_url, + git_branch='master', + git_token=None + ): + """ + + :param git_url: + :param git_branch: + :param git_token: + """ + self._git_url = git_url if git_url.endswith(".git") else git_url + '.git' + self._git_branch = git_branch + self._git_token = git_token + local_dir = os.path.join(sys.prefix, 'sdap', 'conf') + super().__init__(local_dir) + self._repo = None + self._init_local_config_repo() + self._latest_commit_key = self._repo.head.commit.hexsha + + def _pull_remote(self): + o = self._repo.remotes.origin + res = o.pull() + return res[0].commit.hexsha # return the latest commit key + + def _init_local_config_repo(self): + self._repo = Repo.init(self._local_dir) + if len(self._repo.remotes) == 0 or 'origin' not in [r.name for r in self._repo.remotes]: + self._repo.create_remote('origin', self._git_url) + self._repo.git.fetch() + self._repo.git.checkout(self._git_branch) + + def when_updated(self, callback): + + while True: + time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS) + remote_commit_key = self._pull_remote() + if remote_commit_key != self._latest_commit_key: + logger.info("remote git repository has been updated") + callback() + self._latest_commit_key = remote_commit_key + else: + logger.debug("remote git repository has not been updated") + + + + + + diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py index 27a89b8..852920f 100644 --- a/sdap_ingest_manager/config/__init__.py +++ b/sdap_ingest_manager/config/__init__.py @@ -1 +1,4 @@ from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration +from sdap_ingest_manager.config.ConfigMap import ConfigMap +from sdap_ingest_manager.confg.LocalDirConfig import LocalDirConfig +from sdpa_ingest_manager.config.RemoteGitConfig import RemoteGitConfig \ No newline at end of file diff --git a/sdap_ingest_manager/config/exceptions.py b/sdap_ingest_manager/config/exceptions.py new file mode 100644 index 0000000..c06b881 --- /dev/null +++ b/sdap_ingest_manager/config/exceptions.py @@ -0,0 +1,4 @@ + + +class UnreadableFileException(Exception): + pass \ No newline at end of file diff --git a/sdap_ingest_manager/config_operator.py b/sdap_ingest_manager/config_operator.py new file mode 100644 index 0000000..fa2771a --- /dev/null +++ b/sdap_ingest_manager/config_operator.py @@ -0,0 +1,35 @@ +import argparse +from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, ConfigMap + + + +def main(): + parser = argparse.ArgumentParser(description="Run git configuration synchronization operator, work on local-dir or git-url") + input_group = parser.add_mutually_exclusive_group(required=True) + input_group.add_argument("-l", "--local-dir", + help="local directory where the configuration files are") + input_group.add_argument("-gu", "--git-url", + help="git repository from which the configuration files are pulled/saved") + parser.add_argument("-gb", "--git-branch", help="git branch from which the configuration files are pulled/saved", + default="master") + parser.add_argument("-gt", "--git-token", help="git personal access token used to access the repository") + + parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True) + parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True) + + options = parser.parse_args() + + if options.local_dir: + config = LocalDirConfig(options.local_dir) + else: + config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token) + + config_map = ConfigMap(options.config_map, options.namespace, config) + config_map.publish() + + config.when_updated(config_map.publish) + + +if __name__ == "__main__": + main() + diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py index a536202..2518b9a 100644 --- a/tests/config/test_ConfigMap.py +++ b/tests/config/test_ConfigMap.py @@ -5,25 +5,15 @@ from flask import Flask from flask_restplus import Api from sdap_ingest_manager.config.ConfigMap import ConfigMap -from sdap_ingest_manager.ingestion_order_store.FileIngestionOrderStore import FileIngestionOrderStore -from sdap_ingest_manager.ingestion_order_store.templates import Templates +from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig -flask_app = Flask(__name__) -app = Api(app=flask_app) -templates = Templates(app) class ConfigMapTest(unittest.TestCase): def test_createconfigmap(self): - test_ingestion_order_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), - '..', - 'resources', - 'data', - 'collections.yml') - file_ingestion_order_store = FileIngestionOrderStore(path=test_ingestion_order_file, - order_template=templates.order_template) + remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";) - config_map = ConfigMap('collections.yml', 'sdap', file_ingestion_order_store) + config_map = ConfigMap('collection-ingester', 'sdap', remote_git_config) config_map.publish() diff --git a/tests/resources/data/collections.yml b/tests/resources/data/collections.yml index 07e795b..18226ba 100644 --- a/tests/resources/data/collections.yml +++ b/tests/resources/data/collections.yml @@ -1,9 +1,9 @@ avhrr-oi-analysed-sst: path: resources/history_manager/data/avhrr_oi/*.nc variable: analysed_sst - priority: 2 + priority: 8 avhrr-oi-analysed-sst2: path: resources/history_manager/data/avhrr_oi/*.nc variable: analysed_sst - priority: 1 \ No newline at end of file + priority: 1
