This is an automated email from the ASF dual-hosted git repository. jbonofre pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 14716fcab2ad7f1140d14333e6435bc158e779e8 Author: aviemzur <[email protected]> AuthorDate: Mon Mar 16 10:51:56 2020 +0200 Add python_server service --- rainbow/build/build_rainbows.py | 51 ++++++--- .../hello_world => rainbow/build/http}/__init__.py | 0 rainbow/build/http/python/Dockerfile | 24 +++++ rainbow/{http => build/http/python}/__init__.py | 0 .../build/http/python/python_server_image.py | 29 +++-- .../http/python/python_server_requirements.txt | 2 + rainbow/build/http/python/rainbow_python_server.py | 62 +++++++++++ rainbow/build/image_builder.py | 118 +++++++++++++++++++++ rainbow/build/python/python_image.py | 62 ++--------- requirements.txt | 5 +- .../hello_world => build/http}/__init__.py | 0 .../hello_world => build/http/python}/__init__.py | 0 .../build/http/python/test_python_server_image.py | 105 ++++++++++++++++++ .../airflow/build/python/test_python_image.py | 11 +- tests/runners/airflow/build/test_build_rainbow.py | 27 ----- tests/runners/airflow/build/test_build_rainbows.py | 56 ++++++++++ .../{hello_world => helloworld}/__init__.py | 0 .../{hello_world => helloworld}/hello_world.py | 0 .../rainbow/{hello_world => myserver}/__init__.py | 0 .../__init__.py => myserver/my_server.py} | 4 + tests/runners/airflow/rainbow/rainbow.yml | 29 +++-- tests/runners/airflow/tasks/test_python.py | 2 +- 22 files changed, 465 insertions(+), 122 deletions(-) diff --git a/rainbow/build/build_rainbows.py b/rainbow/build/build_rainbows.py index 2a9e6a3..c4a14c7 100644 --- a/rainbow/build/build_rainbows.py +++ b/rainbow/build/build_rainbows.py @@ -20,36 +20,59 @@ import os import yaml +from rainbow.build.http.python.python_server_image import PythonServerImageBuilder +from rainbow.build.python.python_image import PythonImageBuilder from rainbow.core.util import files_util -from rainbow.build.python.python_image import PythonImage def build_rainbows(path): """ - TODO: doc for build_rainbow + TODO: doc for build_rainbows """ - config_files = files_util.find_config_files(path) for config_file in config_files: print(f'Building artifacts for file: {config_file}') + base_path = os.path.dirname(config_file) + with open(config_file) as stream: - config = yaml.safe_load(stream) + rainbow_config = yaml.safe_load(stream) - for pipeline in config['pipelines']: + for pipeline in rainbow_config['pipelines']: for task in pipeline['tasks']: - task_type = task['type'] - task_instance = get_build_class(task_type)() - task_instance.build(base_path=os.path.dirname(config_file), - relative_source_path=task['source'], - tag=task['image']) + builder_class = __get_task_build_class(task['type']) + __build_image(base_path, task, builder_class) + + for service in rainbow_config['services']: + builder_class = __get_service_build_class(service['type']) + __build_image(base_path, service, builder_class) + + +def __build_image(base_path, builder_config, builder): + if 'source' in builder_config: + server_builder_instance = builder( + config=builder_config, + base_path=base_path, + relative_source_path=builder_config['source'], + tag=builder_config['image']) + server_builder_instance.build() + else: + print(f"No source provided for {builder_config['name']}, skipping.") -build_classes = { - 'python': PythonImage +__task_build_classes = { + 'python': PythonImageBuilder, } +__service_build_classes = { + 'python_server': PythonServerImageBuilder +} + + +def __get_task_build_class(task_type): + return __task_build_classes[task_type] + -def get_build_class(task_type): - return build_classes[task_type] +def __get_service_build_class(task_type): + return __service_build_classes[task_type] diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/rainbow/build/http/__init__.py similarity index 100% copy from tests/runners/airflow/rainbow/hello_world/__init__.py copy to rainbow/build/http/__init__.py diff --git a/rainbow/build/http/python/Dockerfile b/rainbow/build/http/python/Dockerfile new file mode 100644 index 0000000..6119437 --- /dev/null +++ b/rainbow/build/http/python/Dockerfile @@ -0,0 +1,24 @@ +# Use an official Python runtime as a parent image +FROM python:3.7-slim + +# Install aptitude build-essential +#RUN apt-get install -y --reinstall build-essential + +# Set the working directory to /app +WORKDIR /app + +# Order of operations is important here for docker's caching & incremental build performance. ! +# Be careful when changing this code. ! + +# Install any needed packages specified in python_server_requirements.txt and requirements.txt +COPY ./python_server_requirements.txt /app +RUN pip install -r python_server_requirements.txt + +COPY ./requirements.txt /app +RUN pip install -r requirements.txt + +# Copy the current directory contents into the container at /app +RUN echo "Copying source code.." +COPY . /app + +CMD python -u rainbow_python_server.py diff --git a/rainbow/http/__init__.py b/rainbow/build/http/python/__init__.py similarity index 100% rename from rainbow/http/__init__.py rename to rainbow/build/http/python/__init__.py diff --git a/tests/runners/airflow/rainbow/hello_world/hello_world.py b/rainbow/build/http/python/python_server_image.py similarity index 51% copy from tests/runners/airflow/rainbow/hello_world/hello_world.py copy to rainbow/build/http/python/python_server_image.py index 3eae465..9a65477 100644 --- a/tests/runners/airflow/rainbow/hello_world/hello_world.py +++ b/rainbow/build/http/python/python_server_image.py @@ -15,13 +15,28 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json -print('Hello world!') -print() +import os -with open('/rainbow_input.json') as file: - print(json.loads(file.readline())) +from rainbow.build.image_builder import ImageBuilder +import yaml -with open('/output.json', 'w') as file: - file.write(json.dumps({'a': 1, 'b': 2})) + +class PythonServerImageBuilder(ImageBuilder): + + def __init__(self, config, base_path, relative_source_path, tag): + super().__init__(config, base_path, relative_source_path, tag) + + @staticmethod + def _dockerfile_path(): + return os.path.join(os.path.dirname(__file__), 'Dockerfile') + + @staticmethod + def _additional_files_from_paths(): + return [ + os.path.join(os.path.dirname(__file__), 'rainbow_python_server.py'), + os.path.join(os.path.dirname(__file__), 'python_server_requirements.txt') + ] + + def _additional_files_from_filename_content_pairs(self): + return [('service.yml', yaml.safe_dump(self.config))] diff --git a/rainbow/build/http/python/python_server_requirements.txt b/rainbow/build/http/python/python_server_requirements.txt new file mode 100644 index 0000000..c395de6 --- /dev/null +++ b/rainbow/build/http/python/python_server_requirements.txt @@ -0,0 +1,2 @@ +Flask==1.1.1 +pyyaml diff --git a/rainbow/build/http/python/rainbow_python_server.py b/rainbow/build/http/python/rainbow_python_server.py new file mode 100644 index 0000000..66aab27 --- /dev/null +++ b/rainbow/build/http/python/rainbow_python_server.py @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import yaml +from flask import Flask + +app = Flask(__name__) + + +def start_server(yml_path): + with open(yml_path) as stream: + __start_server(yaml.safe_load(stream)) + + +def __start_server(config): + endpoints = config['endpoints'] + + for endpoint_config in endpoints: + print(f'Registering endpoint: {endpoint_config}') + endpoint = endpoint_config['endpoint'] + + print(endpoint_config['module']) + + module = __get_module(endpoint_config['module']) + function = module.__getattribute__(endpoint_config['function']) + + app.add_url_rule(rule=endpoint, + endpoint=endpoint, + view_func=function, + methods=['GET', 'POST']) + + print('Starting python server') + + app.run(host='0.0.0.0', threaded=False, port=80) + + +def __get_module(kls): + parts = kls.split('.') + module = ".".join(parts) + m = __import__(module) + for comp in parts[1:]: + m = getattr(m, comp) + return m + + +if __name__ == "__main__": + start_server('service.yml') diff --git a/rainbow/build/image_builder.py b/rainbow/build/image_builder.py new file mode 100644 index 0000000..b54dc00 --- /dev/null +++ b/rainbow/build/image_builder.py @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import shutil +import tempfile + +import docker + + +class ImageBuilder: + """ + Builds an image from source code + """ + + def __init__(self, config, base_path, relative_source_path, tag): + """ + TODO: pydoc + + :param config: + :param base_path: + :param relative_source_path: + :param tag: + """ + self.base_path = base_path + self.relative_source_path = relative_source_path + self.tag = tag + self.config = config + + def build(self): + """ + Builds source code into an image. + """ + print(f'[ ] Building image: {self.tag}') + + temp_dir = self.__temp_dir() + + self.__copy_source_code(temp_dir) + self.__write_additional_files(temp_dir) + + # TODO: log docker output + docker_client = docker.from_env() + docker_client.images.build(path=temp_dir, tag=self.tag) + docker_client.close() + + self.__remove_dir(temp_dir) + + print(f'[X] Building image: {self.tag} (Success).') + + def __copy_source_code(self, temp_dir): + self.__copy_dir(os.path.join(self.base_path, self.relative_source_path), temp_dir) + + def __write_additional_files(self, temp_dir): + # TODO: move requirements.txt related code to a parent class for python image builders. + requirements_file_path = os.path.join(temp_dir, 'requirements.txt') + if not os.path.exists(requirements_file_path): + with open(requirements_file_path, 'w'): + pass + + for file in [self._dockerfile_path()] + self._additional_files_from_paths(): + self.__copy_file(file, temp_dir) + + for filename, content in self._additional_files_from_filename_content_pairs(): + with open(os.path.join(temp_dir, filename), 'w') as file: + file.write(content) + + def __temp_dir(self): + temp_dir = tempfile.mkdtemp() + # Delete dir for shutil.copytree to work + self.__remove_dir(temp_dir) + return temp_dir + + @staticmethod + def __remove_dir(temp_dir): + shutil.rmtree(temp_dir) + + @staticmethod + def __copy_dir(source_path, destination_path): + shutil.copytree(source_path, destination_path) + + @staticmethod + def __copy_file(source_file_path, destination_file_path): + shutil.copy2(source_file_path, destination_file_path) + + @staticmethod + def _dockerfile_path(): + """ + Path to Dockerfile + """ + raise NotImplementedError() + + @staticmethod + def _additional_files_from_paths(): + """ + List of paths to additional files + """ + return [] + + def _additional_files_from_filename_content_pairs(self): + """ + File name and content pairs to create files from + """ + return [] diff --git a/rainbow/build/python/python_image.py b/rainbow/build/python/python_image.py index f0fb3a0..d856b8c 100644 --- a/rainbow/build/python/python_image.py +++ b/rainbow/build/python/python_image.py @@ -17,62 +17,22 @@ # under the License. import os -import shutil -import tempfile -import docker +from rainbow.build.image_builder import ImageBuilder -class PythonImage: +class PythonImageBuilder(ImageBuilder): - def build(self, base_path, relative_source_path, tag): - """ - TODO: pydoc - - :param base_path: - :param relative_source_path: - :param tag: - :param extra_files: - :return: - """ - - print(f'Building image {tag}') - - temp_dir = tempfile.mkdtemp() - # Delete dir for shutil.copytree to work - os.rmdir(temp_dir) - - self.__copy_source(os.path.join(base_path, relative_source_path), temp_dir) - - requirements_file_path = os.path.join(temp_dir, 'requirements.txt') - if not os.path.exists(requirements_file_path): - with open(requirements_file_path, 'w'): - pass - - docker_files = [ - os.path.join(os.path.dirname(__file__), 'Dockerfile'), - os.path.join(os.path.dirname(__file__), 'container-setup.sh'), - os.path.join(os.path.dirname(__file__), 'container-teardown.sh') - ] - - for file in docker_files: - self.__copy_file(file, temp_dir) - - docker_client = docker.from_env() - - # TODO: log docker output - docker_client.images.build(path=temp_dir, tag=tag) - - docker_client.close() - - print(temp_dir, os.listdir(temp_dir)) - - shutil.rmtree(temp_dir) + def __init__(self, config, base_path, relative_source_path, tag): + super().__init__(config, base_path, relative_source_path, tag) @staticmethod - def __copy_source(source_path, destination_path): - shutil.copytree(source_path, destination_path) + def _dockerfile_path(): + return os.path.join(os.path.dirname(__file__), 'Dockerfile') @staticmethod - def __copy_file(source_file_path, destination_file_path): - shutil.copy2(source_file_path, destination_file_path) + def _additional_files_from_paths(): + return [ + os.path.join(os.path.dirname(__file__), 'container-setup.sh'), + os.path.join(os.path.dirname(__file__), 'container-teardown.sh'), + ] diff --git a/requirements.txt b/requirements.txt index 599ab8b..dd1e232 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -botocore -PyYAML +botocore==1.15.21 docker==4.2.0 apache-airflow==1.10.9 docker-pycreds==0.4.0 click==7.1.1 +Flask=1.1.1 +pyyaml \ No newline at end of file diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/tests/runners/airflow/build/http/__init__.py similarity index 100% copy from tests/runners/airflow/rainbow/hello_world/__init__.py copy to tests/runners/airflow/build/http/__init__.py diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/tests/runners/airflow/build/http/python/__init__.py similarity index 100% copy from tests/runners/airflow/rainbow/hello_world/__init__.py copy to tests/runners/airflow/build/http/python/__init__.py diff --git a/tests/runners/airflow/build/http/python/test_python_server_image.py b/tests/runners/airflow/build/http/python/test_python_server_image.py new file mode 100644 index 0000000..fd38c80 --- /dev/null +++ b/tests/runners/airflow/build/http/python/test_python_server_image.py @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import threading +import time +import unittest +import urllib.request +from unittest import TestCase + +import docker + +from rainbow.build.http.python.python_server_image import PythonServerImageBuilder + + +class TestPythonServer(TestCase): + + def setUp(self) -> None: + super().setUp() + self.docker_client = docker.from_env() + self.config = self.__create_conf('my_task') + self.image_name = self.config['image'] + self.__remove_containers() + + def tearDown(self) -> None: + self.__remove_containers() + self.docker_client.close() + + def test_build_python_server(self): + builder = PythonServerImageBuilder(config=self.config, + base_path='tests/runners/airflow/rainbow', + relative_source_path='myserver', + tag=self.image_name) + + builder.build() + + thread = threading.Thread(target=self.__run_container, args=[self.image_name]) + thread.daemon = True + thread.start() + + time.sleep(2) + + server_response = urllib.request.urlopen("http://localhost:9294/myendpoint1";).read() + + self.assertEqual("b'1'", str(server_response)) + + def __remove_containers(self): + print(f'Stopping containers with image: {self.image_name}') + + all_containers = self.docker_client.containers + matching_containers = all_containers.list(filters={'ancestor': self.image_name}) + + for container in matching_containers: + container_id = container.id + print(f'Stopping container {container_id}') + self.docker_client.api.stop(container_id) + print(f'Removing container {container_id}') + self.docker_client.api.remove_container(container_id) + + self.docker_client.containers.prune() + + def __run_container(self, image_name): + try: + print(f'Running container for image: {image_name}') + self.docker_client.containers.run(image_name, ports={'80/tcp': 9294}) + except Exception as err: + print(err) + pass + + @staticmethod + def __create_conf(task_id): + return { + 'task': task_id, + 'cmd': 'foo bar', + 'image': 'rainbow_server_image', + 'source': 'tests/runners/airflow/rainbow/myserver', + 'input_type': 'my_input_type', + 'input_path': 'my_input', + 'output_path': '/my_output.json', + 'endpoints': [ + { + 'endpoint': '/myendpoint1', + 'module': 'my_server', + 'function': 'myendpoint1func' + } + ] + } + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/runners/airflow/build/python/test_python_image.py b/tests/runners/airflow/build/python/test_python_image.py index d190fba..ff4555d 100644 --- a/tests/runners/airflow/build/python/test_python_image.py +++ b/tests/runners/airflow/build/python/test_python_image.py @@ -19,7 +19,7 @@ from unittest import TestCase import docker -from rainbow.build.python.python_image import PythonImage +from rainbow.build.python.python_image import PythonImageBuilder class TestPythonImage(TestCase): @@ -29,7 +29,12 @@ class TestPythonImage(TestCase): image_name = config['image'] - PythonImage().build('tests/runners/airflow/rainbow', 'hello_world', image_name) + builder = PythonImageBuilder(config=config, + base_path='tests/runners/airflow/rainbow', + relative_source_path='helloworld', + tag=image_name) + + builder.build() # TODO: elaborate test of image, validate input/output @@ -54,7 +59,7 @@ class TestPythonImage(TestCase): 'task': task_id, 'cmd': 'foo bar', 'image': 'rainbow_image', - 'source': 'tests/runners/airflow/rainbow/hello_world', + 'source': 'tests/runners/airflow/rainbow/helloworld', 'input_type': 'my_input_type', 'input_path': 'my_input', 'output_path': '/my_output.json' diff --git a/tests/runners/airflow/build/test_build_rainbow.py b/tests/runners/airflow/build/test_build_rainbow.py deleted file mode 100644 index 0817d6c..0000000 --- a/tests/runners/airflow/build/test_build_rainbow.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest -from unittest import TestCase - -import docker -from rainbow.build import build_rainbows - - -class TestBuildRainbow(TestCase): - - def test_build_rainbow(self): - docker_client = docker.client.from_env() - image_names = ['my_static_input_task_image', 'my_task_output_input_task_image'] - - for image_name in image_names: - if len(docker_client.images.list(image_name)) > 0: - docker_client.images.remove(image=image_name) - - build_rainbows.build_rainbows('tests/runners/airflow/rainbow') - - for image_name in image_names: - docker_client.images.get(name=image_name) - - docker_client.close() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/runners/airflow/build/test_build_rainbows.py b/tests/runners/airflow/build/test_build_rainbows.py new file mode 100644 index 0000000..9a4d31c --- /dev/null +++ b/tests/runners/airflow/build/test_build_rainbows.py @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from unittest import TestCase + +import docker + +from rainbow.build import build_rainbows + + +class TestBuildRainbows(TestCase): + + __image_names = [ + 'my_static_input_task_image', + 'my_task_output_input_task_image', + 'my_server_image' + ] + + def setUp(self) -> None: + self.docker_client = docker.client.from_env() + self.__remove_images() + + def tearDown(self) -> None: + self.__remove_images() + self.docker_client.close() + + def __remove_images(self): + for image_name in self.__image_names: + if len(self.docker_client.images.list(image_name)) > 0: + self.docker_client.images.remove(image=image_name) + + def test_build_rainbow(self): + build_rainbows.build_rainbows('tests/runners/airflow/rainbow') + + for image in self.__image_names: + self.docker_client.images.get(image) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/tests/runners/airflow/rainbow/helloworld/__init__.py similarity index 100% copy from tests/runners/airflow/rainbow/hello_world/__init__.py copy to tests/runners/airflow/rainbow/helloworld/__init__.py diff --git a/tests/runners/airflow/rainbow/hello_world/hello_world.py b/tests/runners/airflow/rainbow/helloworld/hello_world.py similarity index 100% rename from tests/runners/airflow/rainbow/hello_world/hello_world.py rename to tests/runners/airflow/rainbow/helloworld/hello_world.py diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/tests/runners/airflow/rainbow/myserver/__init__.py similarity index 100% copy from tests/runners/airflow/rainbow/hello_world/__init__.py copy to tests/runners/airflow/rainbow/myserver/__init__.py diff --git a/tests/runners/airflow/rainbow/hello_world/__init__.py b/tests/runners/airflow/rainbow/myserver/my_server.py similarity index 95% rename from tests/runners/airflow/rainbow/hello_world/__init__.py rename to tests/runners/airflow/rainbow/myserver/my_server.py index 217e5db..a3f0f2c 100644 --- a/tests/runners/airflow/rainbow/hello_world/__init__.py +++ b/tests/runners/airflow/rainbow/myserver/my_server.py @@ -15,3 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + + +def myendpoint1func(): + return '1' diff --git a/tests/runners/airflow/rainbow/rainbow.yml b/tests/runners/airflow/rainbow/rainbow.yml index 2000621..e9f9045 100644 --- a/tests/runners/airflow/rainbow/rainbow.yml +++ b/tests/runners/airflow/rainbow/rainbow.yml @@ -29,14 +29,14 @@ pipelines: type: python description: static input task image: my_static_input_task_image - source: hello_world + source: helloworld env_vars: env1: "a" env2: "b" input_type: static input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]' output_path: /output.json - cmd: python hello_world.py + cmd: python -u helloworld.py # - task: my_parallelized_static_input_task # type: python # description: parallelized static input task @@ -48,31 +48,26 @@ pipelines: # input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]' # split_input: True # executors: 2 -# cmd: python hello_world.py +# cmd: python -u helloworld.py - task: my_task_output_input_task type: python description: parallelized static input task image: my_task_output_input_task_image - source: hello_world + source: helloworld env_vars: env1: "a" env2: "b" input_type: task input_path: my_static_input_task - cmd: python hello_world.py + cmd: python -u helloworld.py services: - service: - name: myserver1 - type: python-server + name: my_python_server + type: python_server description: my python server - artifact-id: myserver1artifactid - source: myserver1logicfolder + image: my_server_image + source: myserver endpoints: - - endpoint: - path: /myendpoint1 - module: mymodule1 - function: myfun1 - - endpoint: - path: /myendpoint2 - module: mymodule2 - function: myfun2 + - endpoint: /myendpoint1 + module: myserver.my_server + function: myendpoint1func diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py index 260f71d..18e6c1a 100644 --- a/tests/runners/airflow/tasks/test_python.py +++ b/tests/runners/airflow/tasks/test_python.py @@ -50,7 +50,7 @@ class TestPythonTask(TestCase): 'task': task_id, 'cmd': 'foo bar', 'image': 'rainbow_image', - 'source': 'tests/runners/airflow/rainbow/hello_world', + 'source': 'tests/runners/airflow/rainbow/helloworld', 'input_type': 'my_input_type', 'input_path': 'my_input', 'output_path': '/my_output.json'
