This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch kopf-operator in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 3001747877d87bab5f90115dc0886986f6686acd Author: thomas loubrieu <[email protected]> AuthorDate: Mon Jul 20 20:02:43 2020 -0700 add unit tests, support username/token --- config_operator/README.md | 2 +- .../config_source/LocalDirConfig.py | 37 +++++++----- .../config_source/RemoteGitConfig.py | 44 +++++++++----- config_operator/config_operator/main.py | 12 +++- config_operator/containers/k8s/git-repo-test.yml | 9 +++ config_operator/requirements.txt | 1 + .../tests/config_source/test_LocalDirConfig.py | 70 ++++++++++++++++++++++ .../tests/config_source/test_RemoteGitConfig.py | 52 ++++++++++++++++ config_operator/tests/k8s/test_K8sConfigMap.py | 18 +++++- .../resources/localDirBadTest/collections.yml | 2 + .../tests/resources/localDirTest/.hidden_file.txt | 1 + .../tests/resources/localDirTest/README.md | 1 + .../tests/resources/localDirTest/collections.yml | 1 + 13 files changed, 215 insertions(+), 35 deletions(-) diff --git a/config_operator/README.md b/config_operator/README.md index 4624a0f..91ca05f 100644 --- a/config_operator/README.md +++ b/config_operator/README.md @@ -25,7 +25,7 @@ The component runs as a kubernetes operator (see containerization section) To publish the docker image on dockerhub do (step necessary for kubernetes deployment): docker login - docker push tloubrieu/sdap-ingest-manager:latest + docker push tloubrieu/config-operator:latest ## Kubernetes diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py index cf95f42..e31a0d7 100644 --- a/config_operator/config_operator/config_source/LocalDirConfig.py +++ b/config_operator/config_operator/config_source/LocalDirConfig.py @@ -1,6 +1,8 @@ +import asyncio import os import time import logging +from functools import partial import yaml from typing import Callable @@ -16,11 +18,14 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1 class LocalDirConfig: def __init__(self, local_dir: str, - update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS): + update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, + update_date_fun=os.path.getmtime): logger.info(f'create config on local dir {local_dir}') self._local_dir = local_dir - self._latest_update = self._get_latest_update() + self._update_date_fun = update_date_fun self._update_every_seconds=update_every_seconds + self._latest_update = self._get_latest_update() + def get_files(self): files = [] @@ -49,28 +54,30 @@ class LocalDirConfig: raise UnreadableFileException(e) except yaml.parser.ParserError as e: raise UnreadableFileException(e) - + except yaml.scanner.ScannerError as e: + raise UnreadableFileException(e) def _get_latest_update(self): - m_times = [os.path.getmtime(root) for root, _, _ in os.walk(self._local_dir)] + m_times = [self._update_date_fun(root) for root, _, _ in os.walk(self._local_dir)] if m_times: - return time.ctime(max(m_times)) + return max(m_times) else: return None - def when_updated(self, callback: Callable[[], None]): + async def when_updated(self, callback: Callable[[], None], loop=None): """ call function callback when the local directory is updated. """ - while True: - time.sleep(self._update_every_seconds) - latest_update = self._get_latest_update() - if latest_update is None or (latest_update > self._latest_update): - logger.info("local config dir has been updated") - callback() - self._latest_update = latest_update - else: - logger.debug("local config dir has not been updated") + if loop is None: + loop = asyncio.get_running_loop() + + latest_update = self._get_latest_update() + if latest_update is None or (latest_update > self._latest_update): + logger.info("local config dir has been updated") + callback() + else: + logger.debug("local config dir has not been updated") + loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop)) return None diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py index 17d8223..2385b17 100644 --- a/config_operator/config_operator/config_source/RemoteGitConfig.py +++ b/config_operator/config_operator/config_source/RemoteGitConfig.py @@ -1,7 +1,8 @@ import logging import os import sys -import time +import asyncio +from functools import partial from git import Repo from typing import Callable from .LocalDirConfig import LocalDirConfig @@ -16,10 +17,11 @@ DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf') class RemoteGitConfig(LocalDirConfig): def __init__(self, git_url: str, git_branch: str = 'master', + git_username: str = None, git_token: str = None, update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS, - local_dir: str = DEFAULT_LOCAL_REPO_DIR - ): + local_dir: str = DEFAULT_LOCAL_REPO_DIR, + repo: Repo = None): """ :param git_url: @@ -27,14 +29,23 @@ class RemoteGitConfig(LocalDirConfig): :param git_token: """ self._git_url = git_url if git_url.endswith(".git") else git_url + '.git' + if git_username and git_token: + self._git_url.replace('https://', f'https://{git_username}:{git_token}') + self._git_url.replace('http://', f'http://{git_username}:{git_token}') + self._git_branch = git_branch self._git_token = git_token if local_dir is None: local_dir = DEFAULT_LOCAL_REPO_DIR self._update_every_seconds = update_every_seconds super().__init__(local_dir, update_every_seconds=self._update_every_seconds) - self._repo = None - self._init_local_config_repo() + + if repo: + self._repo = repo + else: + self._repo = None + self._init_local_config_repo() + self._latest_commit_key = self._pull_remote() def _pull_remote(self): @@ -49,19 +60,22 @@ class RemoteGitConfig(LocalDirConfig): self._repo.git.fetch() self._repo.git.checkout(self._git_branch) - def when_updated(self, callback: Callable[[], None]): + async def when_updated(self, callback: Callable[[], None], loop=None): """ call function callback when the remote git repository is updated. """ - while True: - time.sleep(self._update_every_seconds) - remote_commit_key = self._pull_remote() - if remote_commit_key != self._latest_commit_key: - logger.info("remote git repository has been updated") - callback() - self._latest_commit_key = remote_commit_key - else: - logger.debug("remote git repository has not been updated") + if loop is None: + loop = asyncio.get_running_loop() + + remote_commit_key = self._pull_remote() + if remote_commit_key != self._latest_commit_key: + logger.info("remote git repository has been updated") + callback() + self._latest_commit_key = remote_commit_key + else: + logger.debug("remote git repository has not been updated") + + loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop)) return None diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py index db4dbcb..fac6741 100644 --- a/config_operator/config_operator/main.py +++ b/config_operator/config_operator/main.py @@ -1,4 +1,5 @@ import logging +import asyncio import kopf from config_operator.config_source import RemoteGitConfig from config_operator.k8s import K8sConfigMap @@ -6,10 +7,10 @@ from config_operator.k8s import K8sConfigMap logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + @kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs') def create_fn(body, spec, **kwargs): # Get info from Git Repo Config object - name = body['metadata']['name'] namespace = body['metadata']['namespace'] if 'git-url' not in spec.keys(): @@ -23,7 +24,7 @@ def create_fn(body, spec, **kwargs): logger.info(f'config-map = {config_map}') _kargs = {} - for k in {'git-branch', 'git-token', 'update-every-seconds'}: + for k in {'git-branch', 'git-username', 'git-token', 'update-every-seconds'}: if k in spec: logger.info(f'{k} = {spec[k]}') _kargs[k.replace('-', '_')] = spec[k] @@ -32,7 +33,12 @@ def create_fn(body, spec, **kwargs): config_map = K8sConfigMap(config_map, namespace, config) - config.when_updated(config_map.publish) + asyncio.run(config.when_updated(config_map.publish)) msg = f"configmap {config_map} created from git repo {git_url}" return {'message': msg} + + [email protected]() +def login_fn(**kwargs): + return kopf.login_via_client(**kwargs) diff --git a/config_operator/containers/k8s/git-repo-test.yml b/config_operator/containers/k8s/git-repo-test.yml new file mode 100644 index 0000000..6a98454 --- /dev/null +++ b/config_operator/containers/k8s/git-repo-test.yml @@ -0,0 +1,9 @@ +apiVersion: sdap.apache.org/v1 +kind: gitBasedConfig +metadata: + name: collections-config-gitcfg +spec: + git-url: https://github.com/tloubrieu-jpl/sdap-ingester-config.git + git-branch: master + git-token: whatever + config-map: my-configmap \ No newline at end of file diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt index 5d452e2..84ac622 100644 --- a/config_operator/requirements.txt +++ b/config_operator/requirements.txt @@ -2,3 +2,4 @@ GitPython==3.1.2 kubernetes==11.0 kopf==0.26 + diff --git a/config_operator/tests/config_source/test_LocalDirConfig.py b/config_operator/tests/config_source/test_LocalDirConfig.py new file mode 100644 index 0000000..1c48356 --- /dev/null +++ b/config_operator/tests/config_source/test_LocalDirConfig.py @@ -0,0 +1,70 @@ +import asyncio +import unittest +from unittest.mock import Mock +from unittest.mock import patch +import os +import time +from datetime import datetime +from config_operator.config_source import LocalDirConfig +from config_operator.config_source.exceptions import UnreadableFileException + + + +class MyTestCase(unittest.TestCase): + def test_get_files(self): + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + self.assertEqual(len(files), 1) + self.assertEqual(files[0], 'collections.yml') + + def test_get_good_file_content(self): + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + content = local_dir_config.get_file_content(files[0]) + self.assertEqual(content.strip(), 'test: 1') + + def test_get_bad_file_content(self): + unreadable_file = False + try: + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirBadTest") + local_dir_config = LocalDirConfig(local_dir) + files = local_dir_config.get_files() + content = local_dir_config.get_file_content(files[0]) + except UnreadableFileException as e: + unreadable_file = True + + finally: + self.assertTrue(unreadable_file) + + def test_when_udated(self): + + callback = Mock() + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + + local_dir_config = LocalDirConfig(local_dir, update_every_seconds=0.25, update_date_fun=lambda x: datetime.now().timestamp()) + + asyncio.run(local_dir_config.when_updated(callback)) + + time.sleep(2) + + assert callback.called + + def test_when_not_udated(self): + + callback = Mock() + local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest") + + local_dir_config = LocalDirConfig(local_dir, update_every_seconds=0.25, + update_date_fun=lambda x: 0) + + asyncio.run(local_dir_config.when_updated(callback)) + + time.sleep(2) + + assert not callback.called + + +if __name__ == '__main__': + unittest.main() diff --git a/config_operator/tests/config_source/test_RemoteGitConfig.py b/config_operator/tests/config_source/test_RemoteGitConfig.py new file mode 100644 index 0000000..50596d0 --- /dev/null +++ b/config_operator/tests/config_source/test_RemoteGitConfig.py @@ -0,0 +1,52 @@ +import unittest +from unittest.mock import MagicMock, Mock, patch +import asyncio +import os +import tempfile +import time +from git import Repo +from config_operator.config_source import RemoteGitConfig + +class MyTestCase(unittest.TestCase): + + _latest_commit = 0 + + def _get_origin(self): + commit = Mock() + commit.hexsha = self._latest_commit + self._latest_commit += 1 + + pull_result = Mock() + pull_result.commit = commit + + return [pull_result] + + + def test_when_updated(self): + + origin_branch = Mock() + origin_branch.pull = self._get_origin + + remotes = Mock() + remotes.origin = origin_branch + + repo = Mock() + repo.remotes = remotes + + git_config = RemoteGitConfig(git_url='https://github.com/tloubrieu-jpl/sdap-ingester-config', + update_every_seconds=0.25, + local_dir=os.path.join(tempfile.gettempdir(), 'sdap'), + repo=repo) + + callback = Mock() + + asyncio.run(git_config.when_updated(callback)) + + time.sleep(2) + + assert callback.called + + + +if __name__ == '__main__': + unittest.main() diff --git a/config_operator/tests/k8s/test_K8sConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py index 1660e69..710b3f6 100644 --- a/config_operator/tests/k8s/test_K8sConfigMap.py +++ b/config_operator/tests/k8s/test_K8sConfigMap.py @@ -4,8 +4,14 @@ import os from config_operator.k8s import K8sConfigMap from config_operator.config_source import RemoteGitConfig, LocalDirConfig +if 'GIT_USERNAME' in os.environ: + GIT_USERNAME = os.environ['GIT_USERNAME'] +if 'GIT_TOKEN' in os.environ: + GIT_TOKEN = os.environ['GIT_TOKEN'] + class K8sConfigMapTest(unittest.TestCase): + @unittest.skip('requires remote git') def test_createconfigmapfromgit(self): remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config";) @@ -13,10 +19,20 @@ class K8sConfigMapTest(unittest.TestCase): config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) config_map.publish() + @unittest.skip('requires remote git') + def test_createconfigmapfromgit_with_token(self): + remote_git_config = RemoteGitConfig("https://podaac-git.jpl.nasa.gov:8443/podaac-sdap/deployment-configs.git";, + git_username=GIT_USERNAME, + git_token=GIT_TOKEN) + + config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) + config_map.publish() + def test_createconfigmapfromlocaldir(self): local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', - 'resources') + 'resources', + 'localDirTest') remote_git_config = LocalDirConfig(local_dir) config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config) diff --git a/config_operator/tests/resources/localDirBadTest/collections.yml b/config_operator/tests/resources/localDirBadTest/collections.yml new file mode 100644 index 0000000..7903016 --- /dev/null +++ b/config_operator/tests/resources/localDirBadTest/collections.yml @@ -0,0 +1,2 @@ +test: +test diff --git a/config_operator/tests/resources/localDirTest/.hidden_file.txt b/config_operator/tests/resources/localDirTest/.hidden_file.txt new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/.hidden_file.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/config_operator/tests/resources/localDirTest/README.md b/config_operator/tests/resources/localDirTest/README.md new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/README.md @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/config_operator/tests/resources/localDirTest/collections.yml b/config_operator/tests/resources/localDirTest/collections.yml new file mode 100644 index 0000000..4857bf6 --- /dev/null +++ b/config_operator/tests/resources/localDirTest/collections.yml @@ -0,0 +1 @@ +test: 1
