This is an automated email from the ASF dual-hosted git repository. jbonofre pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 38152de3915fff4cc7c647fe2dcabf32ba38331f Author: aviemzur <[email protected]> AuthorDate: Mon Mar 16 17:13:09 2020 +0200 Class registries --- rainbow/build/build_rainbows.py | 61 +++++++++++++++------- rainbow/build/{http/python => image}/__init__.py | 0 rainbow/build/{ => image}/image_builder.py | 4 ++ rainbow/build/{ => image}/python/Dockerfile | 0 rainbow/build/{http => image/python}/__init__.py | 0 .../build/{ => image}/python/container-setup.sh | 0 .../build/{ => image}/python/container-teardown.sh | 0 .../python_image.py => image/python/python.py} | 2 +- rainbow/build/python/__init__.py | 0 rainbow/build/{http/python => service}/__init__.py | 0 .../python => service/python_server}/Dockerfile | 0 .../python => service/python_server}/__init__.py | 0 .../python_server/python_server.py} | 5 +- .../python_server}/python_server_requirements.txt | 0 .../python_server}/rainbow_python_server.py | 0 .../service_image_builder.py} | 1 + rainbow/core/util/class_util.py | 58 ++++++++++++++++++++ rainbow/runners/airflow/dag/rainbow_dags.py | 20 ++++--- ...mage.py => test_python_server_image_builder.py} | 2 +- ...ython_image.py => test_python_image_builder.py} | 4 +- 20 files changed, 124 insertions(+), 33 deletions(-) diff --git a/rainbow/build/build_rainbows.py b/rainbow/build/build_rainbows.py index c4a14c7..fa3a922 100644 --- a/rainbow/build/build_rainbows.py +++ b/rainbow/build/build_rainbows.py @@ -20,9 +20,8 @@ import os import yaml -from rainbow.build.http.python.python_server_image import PythonServerImageBuilder -from rainbow.build.python.python_image import PythonImageBuilder -from rainbow.core.util import files_util +from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin +from rainbow.core.util import files_util, class_util def build_rainbows(path): @@ -41,38 +40,62 @@ def build_rainbows(path): for pipeline in rainbow_config['pipelines']: for task in pipeline['tasks']: - builder_class = __get_task_build_class(task['type']) - __build_image(base_path, task, builder_class) + task_type = task['type'] + builder_class = __get_task_build_class(task_type) + if builder_class: + __build_image(base_path, task, builder_class) + else: + raise ValueError(f'No such task type: {task_type}') for service in rainbow_config['services']: - builder_class = __get_service_build_class(service['type']) - __build_image(base_path, service, builder_class) + service_type = service['type'] + builder_class = __get_service_build_class(service_type) + if builder_class: + __build_image(base_path, service, builder_class) + else: + raise ValueError(f'No such service type: {service_type}') def __build_image(base_path, builder_config, builder): if 'source' in builder_config: - server_builder_instance = builder( + builder_instance = builder( config=builder_config, base_path=base_path, relative_source_path=builder_config['source'], tag=builder_config['image']) - server_builder_instance.build() + builder_instance.build() else: print(f"No source provided for {builder_config['name']}, skipping.") -__task_build_classes = { - 'python': PythonImageBuilder, -} +def __get_task_build_class(task_type): + return task_build_classes[task_type] if task_type in task_build_classes else None -__service_build_classes = { - 'python_server': PythonServerImageBuilder -} +def __get_service_build_class(service_type): + return service_build_classes[service_type] if service_type in service_build_classes else None -def __get_task_build_class(task_type): - return __task_build_classes[task_type] +print(f'Loading image builder implementations..') + +# TODO: add configuration for user image builders package +image_builders_package = 'rainbow/build/image' +user_image_builders_package = 'TODO: user_image_builders_package' + +task_build_classes = class_util.find_subclasses_in_packages( + [image_builders_package, user_image_builders_package], + ImageBuilder) + +print(f'Finished loading image builder implementations: {task_build_classes}') + +print(f'Loading service image builder implementations..') + +# TODO: add configuration for user service image builders package +service_builders_package = 'rainbow/build/service' +user_service_builders_package = 'TODO: user_service_builders_package' + +service_build_classes = class_util.find_subclasses_in_packages( + [service_builders_package, user_service_builders_package], + ServiceImageBuilderMixin) -def __get_service_build_class(task_type): - return __service_build_classes[task_type] +print(f'Finished loading service image builder implementations: {service_build_classes}') diff --git a/rainbow/build/http/python/__init__.py b/rainbow/build/image/__init__.py similarity index 100% copy from rainbow/build/http/python/__init__.py copy to rainbow/build/image/__init__.py diff --git a/rainbow/build/image_builder.py b/rainbow/build/image/image_builder.py similarity index 98% rename from rainbow/build/image_builder.py rename to rainbow/build/image/image_builder.py index b54dc00..e716b9d 100644 --- a/rainbow/build/image_builder.py +++ b/rainbow/build/image/image_builder.py @@ -116,3 +116,7 @@ class ImageBuilder: File name and content pairs to create files from """ return [] + + +class ServiceImageBuilderMixin(object): + pass diff --git a/rainbow/build/python/Dockerfile b/rainbow/build/image/python/Dockerfile similarity index 100% rename from rainbow/build/python/Dockerfile rename to rainbow/build/image/python/Dockerfile diff --git a/rainbow/build/http/__init__.py b/rainbow/build/image/python/__init__.py similarity index 100% copy from rainbow/build/http/__init__.py copy to rainbow/build/image/python/__init__.py diff --git a/rainbow/build/python/container-setup.sh b/rainbow/build/image/python/container-setup.sh similarity index 100% rename from rainbow/build/python/container-setup.sh rename to rainbow/build/image/python/container-setup.sh diff --git a/rainbow/build/python/container-teardown.sh b/rainbow/build/image/python/container-teardown.sh similarity index 100% rename from rainbow/build/python/container-teardown.sh rename to rainbow/build/image/python/container-teardown.sh diff --git a/rainbow/build/python/python_image.py b/rainbow/build/image/python/python.py similarity index 95% rename from rainbow/build/python/python_image.py rename to rainbow/build/image/python/python.py index d856b8c..f4fb03b 100644 --- a/rainbow/build/python/python_image.py +++ b/rainbow/build/image/python/python.py @@ -18,7 +18,7 @@ import os -from rainbow.build.image_builder import ImageBuilder +from rainbow.build.image.image_builder import ImageBuilder class PythonImageBuilder(ImageBuilder): diff --git a/rainbow/build/python/__init__.py b/rainbow/build/python/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rainbow/build/http/python/__init__.py b/rainbow/build/service/__init__.py similarity index 100% copy from rainbow/build/http/python/__init__.py copy to rainbow/build/service/__init__.py diff --git a/rainbow/build/http/python/Dockerfile b/rainbow/build/service/python_server/Dockerfile similarity index 100% rename from rainbow/build/http/python/Dockerfile rename to rainbow/build/service/python_server/Dockerfile diff --git a/rainbow/build/http/python/__init__.py b/rainbow/build/service/python_server/__init__.py similarity index 100% rename from rainbow/build/http/python/__init__.py rename to rainbow/build/service/python_server/__init__.py diff --git a/rainbow/build/http/python/python_server_image.py b/rainbow/build/service/python_server/python_server.py similarity index 90% rename from rainbow/build/http/python/python_server_image.py rename to rainbow/build/service/python_server/python_server.py index 9a65477..3404abf 100644 --- a/rainbow/build/http/python/python_server_image.py +++ b/rainbow/build/service/python_server/python_server.py @@ -18,11 +18,12 @@ import os -from rainbow.build.image_builder import ImageBuilder import yaml +from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin -class PythonServerImageBuilder(ImageBuilder): + +class PythonServerImageBuilder(ImageBuilder, ServiceImageBuilderMixin): def __init__(self, config, base_path, relative_source_path, tag): super().__init__(config, base_path, relative_source_path, tag) diff --git a/rainbow/build/http/python/python_server_requirements.txt b/rainbow/build/service/python_server/python_server_requirements.txt similarity index 100% rename from rainbow/build/http/python/python_server_requirements.txt rename to rainbow/build/service/python_server/python_server_requirements.txt diff --git a/rainbow/build/http/python/rainbow_python_server.py b/rainbow/build/service/python_server/rainbow_python_server.py similarity index 100% rename from rainbow/build/http/python/rainbow_python_server.py rename to rainbow/build/service/python_server/rainbow_python_server.py diff --git a/rainbow/build/http/__init__.py b/rainbow/build/service/service_image_builder.py similarity index 99% rename from rainbow/build/http/__init__.py rename to rainbow/build/service/service_image_builder.py index 217e5db..3742bcc 100644 --- a/rainbow/build/http/__init__.py +++ b/rainbow/build/service/service_image_builder.py @@ -15,3 +15,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + diff --git a/rainbow/core/util/class_util.py b/rainbow/core/util/class_util.py new file mode 100644 index 0000000..59b8543 --- /dev/null +++ b/rainbow/core/util/class_util.py @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import inspect +import os +import sys + + +def find_subclasses_in_packages(packages, parent_class): + """ + Finds all subclasses of given parent class within given packages + :return: map of module ref -> class + """ + classes = {} + + for package in [a for a in sys.path]: + for root, directories, files in os.walk(package): + for file in files: + file_path = os.path.join(root, file) + if any(p in file_path for p in packages) \ + and file.endswith('.py') \ + and '__pycache__' not in file_path: + spec = importlib.util.spec_from_file_location(file[:-3], file_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + for name, obj in inspect.getmembers(mod): + if inspect.isclass(obj) and not obj.__name__.endswith('Mixin'): + module_name = mod.__name__ + class_name = obj.__name__ + module = root[len(package) + 1:].replace('/', '.') + '.' + module_name + clazz = __get_class(module, class_name) + if issubclass(clazz, parent_class): + classes.update({module_name: clazz}) + + return classes + + +def __get_class(the_module, the_class): + m = __import__(the_module) + for comp in the_module.split('.')[1:] + [the_class]: + m = getattr(m, comp) + return m diff --git a/rainbow/runners/airflow/dag/rainbow_dags.py b/rainbow/runners/airflow/dag/rainbow_dags.py index 92b6d64..15b7d9a 100644 --- a/rainbow/runners/airflow/dag/rainbow_dags.py +++ b/rainbow/runners/airflow/dag/rainbow_dags.py @@ -22,8 +22,8 @@ import yaml from airflow import DAG from airflow.models import Variable -from rainbow.core.util import files_util -from rainbow.runners.airflow.tasks.python import PythonTask +from rainbow.core.util import files_util, class_util +from rainbow.runners.airflow.model.task import Task def register_dags(configs_path): @@ -78,15 +78,19 @@ def register_dags(configs_path): return dags -task_classes = { - 'python': PythonTask -} +print(f'Loading task implementations..') + +# TODO: add configuration for user tasks package +task_package = 'rainbow/runners/airflow/tasks' +user_task_package = 'TODO: user_tasks_package' + +task_classes = class_util.find_subclasses_in_packages([task_package, user_task_package], Task) + +print(f'Finished loading task implementations: {task_classes}') def get_task_class(task_type): return task_classes[task_type] -# TODO: configurable path -path = Variable.get('rainbows_dir') -register_dags(path) +register_dags(Variable.get('rainbows_dir')) diff --git a/tests/runners/airflow/build/http/python/test_python_server_image.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py similarity index 97% rename from tests/runners/airflow/build/http/python/test_python_server_image.py rename to tests/runners/airflow/build/http/python/test_python_server_image_builder.py index fd38c80..3423976 100644 --- a/tests/runners/airflow/build/http/python/test_python_server_image.py +++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py @@ -24,7 +24,7 @@ from unittest import TestCase import docker -from rainbow.build.http.python.python_server_image import PythonServerImageBuilder +from rainbow.build.service.python_server.python_server import PythonServerImageBuilder class TestPythonServer(TestCase): diff --git a/tests/runners/airflow/build/python/test_python_image.py b/tests/runners/airflow/build/python/test_python_image_builder.py similarity index 95% rename from tests/runners/airflow/build/python/test_python_image.py rename to tests/runners/airflow/build/python/test_python_image_builder.py index ff4555d..c8328da 100644 --- a/tests/runners/airflow/build/python/test_python_image.py +++ b/tests/runners/airflow/build/python/test_python_image_builder.py @@ -19,10 +19,10 @@ from unittest import TestCase import docker -from rainbow.build.python.python_image import PythonImageBuilder +from rainbow.build.image.python.python import PythonImageBuilder -class TestPythonImage(TestCase): +class TestPythonImageBuilder(TestCase): def test_build(self): config = self.__create_conf('my_task')
