This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch config_map in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 7a655f049c0191aed40352de9162f9d27f1793fb Author: thomas loubrieu <[email protected]> AuthorDate: Mon Jun 8 19:48:44 2020 -0700 apply change requests by pull request #3 review --- config_operator/README.md | 2 ++ .../config_source/LocalDirConfig.py | 18 +++++++++---- .../config_source/RemoteGitConfig.py | 30 +++++++++++++--------- .../config_operator/k8s/K8sConfigMap.py | 13 +++++++--- config_operator/config_operator/main.py | 14 +++++++--- config_operator/containers/docker/Dockerfile | 4 +-- .../containers/k8s/config-operator-crd.yml | 2 ++ .../tests/{config => config_source}/__init__.py | 0 config_operator/tests/{config => k8s}/__init__.py | 0 .../test_ConfigMap.py => k8s/test_K8sConfigMap.py} | 2 +- 10 files changed, 58 insertions(+), 27 deletions(-) diff --git a/config_operator/README.md b/config_operator/README.md index 2e1ea0d..2ff8072 100644 --- a/config_operator/README.md +++ b/config_operator/README.md @@ -51,4 +51,6 @@ Check that the configMap has been generated: kubectl get configmaps -n sdap + + \ No newline at end of file diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py index f37e41b..cf95f42 100644 --- a/config_operator/config_operator/config_source/LocalDirConfig.py +++ b/config_operator/config_operator/config_source/LocalDirConfig.py @@ -2,6 +2,7 @@ import os import time import logging import yaml +from typing import Callable from config_operator.config_source.exceptions import UnreadableFileException @@ -14,10 +15,12 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1 class LocalDirConfig: - def __init__(self, local_dir): + def __init__(self, local_dir: str, + update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS): logger.info(f'create config on local dir {local_dir}') self._local_dir = local_dir self._latest_update = self._get_latest_update() + self._update_every_seconds=update_every_seconds def get_files(self): files = [] @@ -29,14 +32,14 @@ class LocalDirConfig: return files - def _test_read_yaml(self, file_name): + def _test_read_yaml(self, file_name: str): """ check yaml syntax raise yaml.parser.ParserError is it doesn't""" with open(os.path.join(self._local_dir, file_name), 'r') as f: docs = yaml.load_all(f, Loader=yaml.FullLoader) for doc in docs: pass - def get_file_content(self, file_name): + def get_file_content(self, file_name: str): logger.info(f'read configuration file {file_name}') try: self._test_read_yaml(file_name) @@ -55,9 +58,12 @@ class LocalDirConfig: else: return None - def when_updated(self, callback): + def when_updated(self, callback: Callable[[], None]): + """ + call function callback when the local directory is updated. + """ while True: - time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS) + time.sleep(self._update_every_seconds) latest_update = self._get_latest_update() if latest_update is None or (latest_update > self._latest_update): logger.info("local config dir has been updated") @@ -66,3 +72,5 @@ class LocalDirConfig: else: logger.debug("local config dir has not been updated") + return None + diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py index 15c0f01..17d8223 100644 --- a/config_operator/config_operator/config_source/RemoteGitConfig.py +++ b/config_operator/config_operator/config_source/RemoteGitConfig.py @@ -3,6 +3,7 @@ import os import sys import time from git import Repo +from typing import Callable from .LocalDirConfig import LocalDirConfig logging.basicConfig(level=logging.DEBUG) @@ -11,11 +12,13 @@ logger = logging.getLogger(__name__) LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5 DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf') + class RemoteGitConfig(LocalDirConfig): - def __init__(self, git_url, - branch='master', - token=None, - local_dir=DEFAULT_LOCAL_REPO_DIR + def __init__(self, git_url: str, + git_branch: str = 'master', + git_token: str = None, + update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, + local_dir: str = DEFAULT_LOCAL_REPO_DIR ): """ @@ -24,11 +27,12 @@ class RemoteGitConfig(LocalDirConfig): :param git_token: """ self._git_url = git_url if git_url.endswith(".git") else git_url + '.git' - self._git_branch = branch - self._git_token = token + self._git_branch = git_branch + self._git_token = git_token if local_dir is None: local_dir = DEFAULT_LOCAL_REPO_DIR - super().__init__(local_dir) + self._update_every_seconds = update_every_seconds + super().__init__(local_dir, update_every_seconds=self._update_every_seconds) self._repo = None self._init_local_config_repo() self._latest_commit_key = self._pull_remote() @@ -45,12 +49,12 @@ class RemoteGitConfig(LocalDirConfig): self._repo.git.fetch() self._repo.git.checkout(self._git_branch) - - - def when_updated(self, callback): - + def when_updated(self, callback: Callable[[], None]): + """ + call function callback when the remote git repository is updated. + """ while True: - time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS) + time.sleep(self._update_every_seconds) remote_commit_key = self._pull_remote() if remote_commit_key != self._latest_commit_key: logger.info("remote git repository has been updated") @@ -59,3 +63,5 @@ class RemoteGitConfig(LocalDirConfig): else: logger.debug("remote git repository has not been updated") + return None + diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py index 32ee844..e2a7a10 100644 --- a/config_operator/config_operator/k8s/K8sConfigMap.py +++ b/config_operator/config_operator/k8s/K8sConfigMap.py @@ -1,7 +1,9 @@ import os import logging from kubernetes import client, config +from config_operator.config_source import LocalDirConfig, RemoteGitConfig from kubernetes.client.rest import ApiException +from typing import Union from config_operator.config_source.exceptions import UnreadableFileException @@ -10,11 +12,14 @@ logger = logging.getLogger(__name__) class K8sConfigMap: - def __init__(self, configmap_name, namespace, git_remote_config): - self._git_remote_config = git_remote_config + def __init__(self, configmap_name: str, + namespace: str, + external_config: Union[LocalDirConfig, RemoteGitConfig]): + self._git_remote_config = external_config self._namespace = namespace self._configmap_name = configmap_name + # test is this runs inside kubernetes cluster if os.getenv('KUBERNETES_SERVICE_HOST'): config.load_incluster_config() else: @@ -93,6 +98,8 @@ class K8sConfigMap: try: self._create() except ApiException as e: - logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e) + logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to patch %s\n" % e) self._patch() + return None + diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py index 3d6ad2e..db4dbcb 100644 --- a/config_operator/config_operator/main.py +++ b/config_operator/config_operator/main.py @@ -1,9 +1,12 @@ +import logging import kopf from config_operator.config_source import RemoteGitConfig from config_operator.k8s import K8sConfigMap +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) [email protected]('sdap.apache.org', 'v1', 'git-repo-configs') [email protected]('sdap.apache.org', 'v1', 'gitbasedconfigs') def create_fn(body, spec, **kwargs): # Get info from Git Repo Config object name = body['metadata']['name'] @@ -15,12 +18,15 @@ def create_fn(body, spec, **kwargs): raise kopf.HandlerFatalError(f"config-map must be set.") git_url = spec['git-url'] + logger.info(f'git-url = {git_url}') config_map = spec['config-map'] + logger.info(f'config-map = {config_map}') _kargs = {} - for k in {'git-branch', 'git-token'}: - if k in spec.keys(): - _kargs[k.split('-')[0]] = spec[k] + for k in {'git-branch', 'git-token', 'update-every-seconds'}: + if k in spec: + logger.info(f'{k} = {spec[k]}') + _kargs[k.replace('-', '_')] = spec[k] config = RemoteGitConfig(git_url, **_kargs) diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile index 4765d30..a161dae 100644 --- a/config_operator/containers/docker/Dockerfile +++ b/config_operator/containers/docker/Dockerfile @@ -6,6 +6,6 @@ COPY /requirements.txt /config_operator/requirements.txt COPY /README.md /config_operator/README.md RUN cd /config_operator && pip install . -COPY /config_operator/k8_config_operator.py /k8_config_operator.py +COPY /config_operator/main.py /main.py -CMD ["kopf", "run", "/k8_config_operator.py", "--verbose"] +CMD ["kopf", "run", "/main.py", "--verbose"] diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml index ea9bbba..9f6d2ad 100644 --- a/config_operator/containers/k8s/config-operator-crd.yml +++ b/config_operator/containers/k8s/config-operator-crd.yml @@ -25,6 +25,8 @@ spec: type: string git-token: type: string + update-every-seconds: + type: int local-dir: type: string namespace: diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/config_source/__init__.py similarity index 100% copy from config_operator/tests/config/__init__.py copy to config_operator/tests/config_source/__init__.py diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/k8s/__init__.py similarity index 100% rename from config_operator/tests/config/__init__.py rename to config_operator/tests/k8s/__init__.py diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py similarity index 95% rename from config_operator/tests/config/test_ConfigMap.py rename to config_operator/tests/k8s/test_K8sConfigMap.py index 4d339e3..1660e69 100644 --- a/config_operator/tests/config/test_ConfigMap.py +++ b/config_operator/tests/k8s/test_K8sConfigMap.py @@ -5,7 +5,7 @@ from config_operator.k8s import K8sConfigMap from config_operator.config_source import RemoteGitConfig, LocalDirConfig -class ConfigMapTest(unittest.TestCase): +class K8sConfigMapTest(unittest.TestCase): def test_createconfigmapfromgit(self): remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";)
