This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch ascending_latitudes in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 5256cddee70efa2ac2869ce4229b4792f20ab42c Author: thomas loubrieu <[email protected]> AuthorDate: Wed Jul 29 17:54:25 2020 -0700 SDAP-247: config-operator unit tests and support for git username/token (#6) Co-authored-by: thomas loubrieu <[email protected]> Co-authored-by: Eamon Ford <[email protected]> --- config_operator/README.md | 6 +- .../config_source/LocalDirConfig.py | 40 ++++++------ .../config_source/RemoteGitConfig.py | 51 +++++++++------ .../config_operator/k8s/K8sConfigMap.py | 26 +++++--- config_operator/config_operator/main.py | 12 +++- config_operator/containers/k8s/git-repo-test.yml | 9 +++ config_operator/requirements.txt | 1 + .../tests/config_source/test_LocalDirConfig.py | 72 ++++++++++++++++++++++ .../tests/config_source/test_RemoteGitConfig.py | 49 +++++++++++++++ config_operator/tests/k8s/test_K8sConfigMap.py | 53 +++++++++++++++- .../resources/localDirBadTest/collections.yml | 2 + .../tests/resources/localDirTest/.hidden_file.txt | 1 + .../tests/resources/localDirTest/README.md | 1 + .../tests/resources/localDirTest/collections.yml | 1 + 14 files changed, 269 insertions(+), 55 deletions(-) diff --git a/config_operator/README.md b/config_operator/README.md index 4624a0f..ba4c6fc 100644 --- a/config_operator/README.md +++ b/config_operator/README.md @@ -16,16 +16,16 @@ The component runs as a kubernetes operator (see containerization section) pip install -e . pytest -d -# Containerizaion +# Containerization ## Docker - docker build . -f containers/docker/Dockerfile --no-cache --tag tloubrieu/config-operator:latest + docker build . -f containers/docker/Dockerfile -t nexusjpl/config-operator:latest To publish the docker image on dockerhub do (step necessary for kubernetes deployment): docker login - docker push tloubrieu/sdap-ingest-manager:latest + docker push nexusjpl/config-operator:latest ## Kubernetes diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py index cf95f42..555a7fa 100644 --- a/config_operator/config_operator/config_source/LocalDirConfig.py +++ b/config_operator/config_operator/config_source/LocalDirConfig.py @@ -1,10 +1,11 @@ +import asyncio import os import time import logging +from functools import partial import yaml from typing import Callable - from config_operator.config_source.exceptions import UnreadableFileException logging.basicConfig(level=logging.DEBUG) @@ -16,12 +17,14 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1 class LocalDirConfig: def __init__(self, local_dir: str, - update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS): + update_every_seconds: float = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, + update_date_fun=os.path.getmtime): logger.info(f'create config on local dir {local_dir}') self._local_dir = local_dir + self._update_date_fun = update_date_fun + self._update_every_seconds = update_every_seconds self._latest_update = self._get_latest_update() - self._update_every_seconds=update_every_seconds - + def get_files(self): files = [] for f in os.listdir(self._local_dir): @@ -49,28 +52,29 @@ class LocalDirConfig: raise UnreadableFileException(e) except yaml.parser.ParserError as e: raise UnreadableFileException(e) - + except yaml.scanner.ScannerError as e: + raise UnreadableFileException(e) def _get_latest_update(self): - m_times = [os.path.getmtime(root) for root, _, _ in os.walk(self._local_dir)] + m_times = [self._update_date_fun(root) for root, _, _ in os.walk(self._local_dir)] if m_times: - return time.ctime(max(m_times)) + return max(m_times) else: return None - def when_updated(self, callback: Callable[[], None]): + async def when_updated(self, callback: Callable[[], None], loop=None): """ call function callback when the local directory is updated. """ - while True: - time.sleep(self._update_every_seconds) - latest_update = self._get_latest_update() - if latest_update is None or (latest_update > self._latest_update): - logger.info("local config dir has been updated") - callback() - self._latest_update = latest_update - else: - logger.debug("local config dir has not been updated") + if loop is None: + loop = asyncio.get_running_loop() - return None + latest_update = self._get_latest_update() + if latest_update is None or (latest_update > self._latest_update): + logger.info("local config dir has been updated") + callback() + else: + logger.debug("local config dir has not been updated") + loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop)) + return None diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py index 17d8223..38cbe2d 100644 --- a/config_operator/config_operator/config_source/RemoteGitConfig.py +++ b/config_operator/config_operator/config_source/RemoteGitConfig.py @@ -1,9 +1,12 @@ +import asyncio import logging import os import sys -import time -from git import Repo +from functools import partial from typing import Callable + +from git import Repo + from .LocalDirConfig import LocalDirConfig logging.basicConfig(level=logging.DEBUG) @@ -16,10 +19,11 @@ DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf') class RemoteGitConfig(LocalDirConfig): def __init__(self, git_url: str, git_branch: str = 'master', + git_username: str = None, git_token: str = None, - update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, - local_dir: str = DEFAULT_LOCAL_REPO_DIR - ): + update_every_seconds: float = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, + local_dir: str = DEFAULT_LOCAL_REPO_DIR, + repo: Repo = None): """ :param git_url: @@ -27,14 +31,23 @@ class RemoteGitConfig(LocalDirConfig): :param git_token: """ self._git_url = git_url if git_url.endswith(".git") else git_url + '.git' + if git_username and git_token: + self._git_url.replace('https://', f'https://{git_username}:{git_token}') + self._git_url.replace('http://', f'http://{git_username}:{git_token}') + self._git_branch = git_branch self._git_token = git_token if local_dir is None: local_dir = DEFAULT_LOCAL_REPO_DIR self._update_every_seconds = update_every_seconds super().__init__(local_dir, update_every_seconds=self._update_every_seconds) - self._repo = None - self._init_local_config_repo() + + if repo: + self._repo = repo + else: + self._repo = None + self._init_local_config_repo() + self._latest_commit_key = self._pull_remote() def _pull_remote(self): @@ -49,19 +62,21 @@ class RemoteGitConfig(LocalDirConfig): self._repo.git.fetch() self._repo.git.checkout(self._git_branch) - def when_updated(self, callback: Callable[[], None]): + async def when_updated(self, callback: Callable[[], None], loop=None): """ call function callback when the remote git repository is updated. """ - while True: - time.sleep(self._update_every_seconds) - remote_commit_key = self._pull_remote() - if remote_commit_key != self._latest_commit_key: - logger.info("remote git repository has been updated") - callback() - self._latest_commit_key = remote_commit_key - else: - logger.debug("remote git repository has not been updated") + if loop is None: + loop = asyncio.get_running_loop() - return None + remote_commit_key = self._pull_remote() + if remote_commit_key != self._latest_commit_key: + logger.info("remote git repository has been updated") + callback() + self._latest_commit_key = remote_commit_key + else: + logger.debug("remote git repository has not been updated") + loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop)) + + return None diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py index e2a7a10..40d07c9 100644 --- a/config_operator/config_operator/k8s/K8sConfigMap.py +++ b/config_operator/config_operator/k8s/K8sConfigMap.py @@ -4,7 +4,8 @@ from kubernetes import client, config from config_operator.config_source import LocalDirConfig, RemoteGitConfig from kubernetes.client.rest import ApiException from typing import Union - +from kubernetes.client.api.core_v1_api import CoreV1Api +from kubernetes.client import ApiClient from config_operator.config_source.exceptions import UnreadableFileException logging.basicConfig(level=logging.INFO) @@ -14,19 +15,24 @@ logger = logging.getLogger(__name__) class K8sConfigMap: def __init__(self, configmap_name: str, namespace: str, - external_config: Union[LocalDirConfig, RemoteGitConfig]): + external_config: Union[LocalDirConfig, RemoteGitConfig], + api_instance: ApiClient = None, + api_core_v1_instance: CoreV1Api = None): self._git_remote_config = external_config self._namespace = namespace self._configmap_name = configmap_name - # test is this runs inside kubernetes cluster - if os.getenv('KUBERNETES_SERVICE_HOST'): - config.load_incluster_config() - else: - config.load_kube_config() - configuration = client.Configuration() - self._api_instance = client.ApiClient(configuration) - self._api_core_v1_instance = client.CoreV1Api(self._api_instance) + if api_core_v1_instance is None: + # test is this runs inside kubernetes cluster + if os.getenv('KUBERNETES_SERVICE_HOST'): + config.load_incluster_config() + else: + config.load_kube_config() + configuration = client.Configuration() + api_instance = client.ApiClient(configuration) + api_core_v1_instance = client.CoreV1Api(api_instance) + self._api_instance = api_instance + self._api_core_v1_instance = api_core_v1_instance self.publish() def __del__(self): diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py index db4dbcb..fac6741 100644 --- a/config_operator/config_operator/main.py +++ b/config_operator/config_operator/main.py @@ -1,4 +1,5 @@ import logging +import asyncio import kopf from config_operator.config_source import RemoteGitConfig from config_operator.k8s import K8sConfigMap @@ -6,10 +7,10 @@ from config_operator.k8s import K8sConfigMap logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + @kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs') def create_fn(body, spec, **kwargs): # Get info from Git Repo Config object - name = body['metadata']['name'] namespace = body['metadata']['namespace'] if 'git-url' not in spec.keys(): @@ -23,7 +24,7 @@ def create_fn(body, spec, **kwargs): logger.info(f'config-map = {config_map}') _kargs = {} - for k in {'git-branch', 'git-token', 'update-every-seconds'}: + for k in {'git-branch', 'git-username', 'git-token', 'update-every-seconds'}: if k in spec: logger.info(f'{k} = {spec[k]}') _kargs[k.replace('-', '_')] = spec[k] @@ -32,7 +33,12 @@ def create_fn(body, spec, **kwargs): config_map = K8sConfigMap(config_map, namespace, config) - config.when_updated(config_map.publish) + asyncio.run(config.when_updated(config_map.publish)) msg = f"configmap {config_map} created from git repo {git_url}" return {'message': msg} + + [email protected]() +def login_fn(**kwargs): + return kopf.login_via_client(**kwargs) diff --git a/config_operator/containers/k8s/git-repo-test.yml b/config_operator/containers/k8s/git-repo-test.yml new file mode 100644 index 0000000..6a98454 --- /dev/null +++ b/config_operator/containers/k8s/git-repo-test.yml @@ -0,0 +1,9 @@ +apiVersion: sdap.apache.org/v1 +kind: gitBasedConfig +metadata: + name: collections-config-gitcfg +spec: + git-url: https://github.com/tloubrieu-jpl/sdap-ingester-config.git + git-branch: master + git-token: whatever + config-map: my-configmap \ No newline at end of file diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt index 5d452e2..84ac622 100644 --- a/config_operator/requirements.txt +++ b/config_operator/requirements.txt @@ -2,3 +2,4 @@ GitPython==3.1.2 kubernetes==11.0 kopf==0.26 + diff --git a/config_operator/tests/config_source/test_LocalDirConfig.py b/config_operator/tests/config_source/test_LocalDirConfig.py new file mode 100644 index 0000000..fed13c0 --- /dev/null +++ b/config_operator/tests/config_source/test_LocalDirConfig.py @@ -0,0 +1,72 @@ +import asyncio +import os +import time +import unittest +from datetime import datetime +from unittest.mock import Mock + +from config_operator.config_source import LocalDirConfig +from config_operator.config_source.exceptions import UnreadableFileException + + +class TestLocalDirConfig(unittest.TestCase): + def test_get_files(self): + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + self.assertEqual(len(files), 1) + self.assertEqual(files[0], 'collections.yml') + + def test_get_good_file_content(self): + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + content = local_dir_config.get_file_content(files[0]) + self.assertEqual(content.strip(), 'test: 1') + + def test_get_bad_file_content(self): + unreadable_file = False + try: + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirBadTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + content = local_dir_config.get_file_content(files[0]) + except UnreadableFileException as e: + unreadable_file = True + + finally: + self.assertTrue(unreadable_file) + + def test_when_updated(self): + + callback = Mock() + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + + local_dir_config = LocalDirConfig(local_dir, + update_every_seconds=0.25, + update_date_fun=lambda x: datetime.now().timestamp()) + + asyncio.run(local_dir_config.when_updated(callback)) + + time.sleep(2) + + assert callback.called + + def test_when_not_updated(self): + + callback = Mock() + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + + local_dir_config = LocalDirConfig(local_dir, + update_every_seconds=0.25, + update_date_fun=lambda x: 0) + + asyncio.run(local_dir_config.when_updated(callback)) + + time.sleep(2) + + assert not callback.called + + +if __name__ == '__main__': + unittest.main() diff --git a/config_operator/tests/config_source/test_RemoteGitConfig.py b/config_operator/tests/config_source/test_RemoteGitConfig.py new file mode 100644 index 0000000..52e6c9d --- /dev/null +++ b/config_operator/tests/config_source/test_RemoteGitConfig.py @@ -0,0 +1,49 @@ +import asyncio +import os +import tempfile +import time +import unittest +from unittest.mock import Mock + +from config_operator.config_source import RemoteGitConfig + + +class TestRemoteDirConfig(unittest.TestCase): + _latest_commit = 0 + + def _get_origin(self): + commit = Mock() + commit.hexsha = self._latest_commit + self._latest_commit += 1 + + pull_result = Mock() + pull_result.commit = commit + + return [pull_result] + + def test_when_updated(self): + origin_branch = Mock() + origin_branch.pull = self._get_origin + + remotes = Mock() + remotes.origin = origin_branch + + repo = Mock() + repo.remotes = remotes + + git_config = RemoteGitConfig(git_url='https://github.com/tloubrieu-jpl/sdap-ingester-config', + update_every_seconds=0.25, + local_dir=os.path.join(tempfile.gettempdir(), 'sdap'), + repo=repo) + + callback = Mock() + + asyncio.run(git_config.when_updated(callback)) + + time.sleep(2) + + assert callback.called + + +if __name__ == '__main__': + unittest.main() diff --git a/config_operator/tests/k8s/test_K8sConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py index 1660e69..e649fbc 100644 --- a/config_operator/tests/k8s/test_K8sConfigMap.py +++ b/config_operator/tests/k8s/test_K8sConfigMap.py @@ -1,11 +1,19 @@ import unittest +from unittest.mock import Mock import os +from kubernetes.client.rest import ApiException from config_operator.k8s import K8sConfigMap from config_operator.config_source import RemoteGitConfig, LocalDirConfig +if 'GIT_USERNAME' in os.environ: + GIT_USERNAME = os.environ['GIT_USERNAME'] +if 'GIT_TOKEN' in os.environ: + GIT_TOKEN = os.environ['GIT_TOKEN'] + class K8sConfigMapTest(unittest.TestCase): + @unittest.skip('requires remote git') def test_createconfigmapfromgit(self): remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";) @@ -13,13 +21,52 @@ class K8sConfigMapTest(unittest.TestCase): config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) config_map.publish() + @unittest.skip('requires remote git') + def test_createconfigmapfromgit_with_token(self): + remote_git_config = RemoteGitConfig("https://podaac-git.jpl.nasa.gov:8443/podaac-sdap/deployment-configs.git";, + git_username=GIT_USERNAME, + git_token=GIT_TOKEN) + + config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) + config_map.publish() + + @unittest.skip('requires kubernetes') + def test_createconfigmapfromlocaldir_with_k8s(self): + local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', + 'resources', + 'localDirTest') + local_config = LocalDirConfig(local_dir) + + config_map = K8sConfigMap('collection-ingester', 'sdap', local_config) + config_map.publish() + def test_createconfigmapfromlocaldir(self): local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', - 'resources') - remote_git_config = LocalDirConfig(local_dir) + 'resources', + 'localDirTest') + local_config = LocalDirConfig(local_dir) - config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) + api_instance = Mock() + api_instance.close = Mock() + + api_core_v1_mock = Mock() + api_core_v1_mock.create_namespaced_config_map = Mock(return_value={ + 'api_version': 'v1', + 'binary_data': None, + 'data': {} + }) + api_core_v1_mock.patch_namespaced_config_map = Mock(return_value={ + 'api_version': 'v1', + 'binary_data': None, + 'data': {} + }) + api_core_v1_mock.create_namespaced_config_map.side_effect = Mock(side_effect=ApiException('409')) + + config_map = K8sConfigMap('collection-ingester', 'sdap', local_config, + api_instance = api_instance, + api_core_v1_instance=api_core_v1_mock) config_map.publish() diff --git a/config_operator/tests/resources/localDirBadTest/collections.yml b/config_operator/tests/resources/localDirBadTest/collections.yml new file mode 100644 index 0000000..7903016 --- /dev/null +++ b/config_operator/tests/resources/localDirBadTest/collections.yml @@ -0,0 +1,2 @@ +test: +test diff --git a/config_operator/tests/resources/localDirTest/.hidden_file.txt b/config_operator/tests/resources/localDirTest/.hidden_file.txt new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/.hidden_file.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/config_operator/tests/resources/localDirTest/README.md b/config_operator/tests/resources/localDirTest/README.md new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/README.md @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/config_operator/tests/resources/localDirTest/collections.yml b/config_operator/tests/resources/localDirTest/collections.yml new file mode 100644 index 0000000..4857bf6 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/collections.yml @@ -0,0 +1 @@ +test: 1
