This is an automated email from the ASF dual-hosted git repository. jbonofre pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 8df40b2cb91eb2428e58c58e7dad9a290f9c185e Author: aviem-naturalint <[email protected]> AuthorDate: Sat Apr 11 08:14:08 2020 +0300 Use user pip conf in docker build --- rainbow/build/build_rainbows.py | 4 +- rainbow/build/image/python/Dockerfile | 26 ++++++- rainbow/build/image/python/container-setup.sh | 2 + rainbow/build/image/python/container-teardown.sh | 4 +- rainbow/build/image/python/python.py | 7 +- rainbow/build/{image => }/image_builder.py | 70 ++++++++++++----- rainbow/build/python.py | 74 ++++++++++++++++++ rainbow/build/service/python_server/Dockerfile | 26 ++++++- .../build/service/python_server/python_server.py | 8 +- .../kubernetes_pod_operator_with_input_output.py | 5 +- run_tests.sh | 4 +- .../python/test_python_server_image_builder.py | 36 +++++++-- .../build/python/test_python_image_builder.py | 90 ++++++++++++++++++---- tests/runners/airflow/build/test_build_rainbows.py | 2 +- .../airflow/rainbow/helloworld/hello_world.py | 10 ++- .../{helloworld/hello_world.py => pip.conf} | 10 --- tests/runners/airflow/rainbow/rainbow.yml | 8 +- 17 files changed, 308 insertions(+), 78 deletions(-) diff --git a/rainbow/build/build_rainbows.py b/rainbow/build/build_rainbows.py index 4ed5bab..b7ea6eb 100644 --- a/rainbow/build/build_rainbows.py +++ b/rainbow/build/build_rainbows.py @@ -20,13 +20,13 @@ import os import yaml -from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin +from rainbow.build.image_builder import ImageBuilder, ServiceImageBuilderMixin from rainbow.core.util import files_util, class_util def build_rainbows(path): """ - TODO: doc for build_rainbows + Build images for rainbows in path. """ config_files = files_util.find_config_files(path) diff --git a/rainbow/build/image/python/Dockerfile b/rainbow/build/image/python/Dockerfile index d4e3ed2..8e4de05 100644 --- a/rainbow/build/image/python/Dockerfile +++ b/rainbow/build/image/python/Dockerfile @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Use an official Python runtime as a parent image FROM python:3.7-slim @@ -11,9 +29,11 @@ WORKDIR /app # Be careful when changing this code. ! # Install any needed packages specified in requirements.txt -COPY ./requirements.txt /app -RUN pip install -r requirements.txt +COPY ./requirements.txt /app/ + +# mount the secret in the correct location, then run pip install +RUN {{mount}} pip install -r requirements.txt # Copy the current directory contents into the container at /app RUN echo "Copying source code.." -COPY . /app +COPY . /app/ diff --git a/rainbow/build/image/python/container-setup.sh b/rainbow/build/image/python/container-setup.sh index 883f1e1..c9e5cef 100755 --- a/rainbow/build/image/python/container-setup.sh +++ b/rainbow/build/image/python/container-setup.sh @@ -1,5 +1,7 @@ #!/bin/sh +echo 'Writing rainbow input..' + echo """$RAINBOW_INPUT""" > /rainbow_input.json AIRFLOW_RETURN_FILE=/airflow/xcom/return.json diff --git a/rainbow/build/image/python/container-teardown.sh b/rainbow/build/image/python/container-teardown.sh index ef213a8..46c4426 100755 --- a/rainbow/build/image/python/container-teardown.sh +++ b/rainbow/build/image/python/container-teardown.sh @@ -1,6 +1,8 @@ #!/bin/sh +echo 'Writing rainbow output..' + USER_CONFIG_OUTPUT_FILE=$1 if [ "$USER_CONFIG_OUTPUT_FILE" != "" ]; then - cp ${USER_CONFIG_OUTPUT_FILE} /airflow/xcom/return.json + cp "${USER_CONFIG_OUTPUT_FILE}" /airflow/xcom/return.json fi diff --git a/rainbow/build/image/python/python.py b/rainbow/build/image/python/python.py index f4fb03b..0ecec77 100644 --- a/rainbow/build/image/python/python.py +++ b/rainbow/build/image/python/python.py @@ -18,10 +18,10 @@ import os -from rainbow.build.image.image_builder import ImageBuilder +from rainbow.build.python import BasePythonImageBuilder -class PythonImageBuilder(ImageBuilder): +class PythonImageBuilder(BasePythonImageBuilder): def __init__(self, config, base_path, relative_source_path, tag): super().__init__(config, base_path, relative_source_path, tag) @@ -30,8 +30,7 @@ class PythonImageBuilder(ImageBuilder): def _dockerfile_path(): return os.path.join(os.path.dirname(__file__), 'Dockerfile') - @staticmethod - def _additional_files_from_paths(): + def _additional_files_from_paths(self): return [ os.path.join(os.path.dirname(__file__), 'container-setup.sh'), os.path.join(os.path.dirname(__file__), 'container-teardown.sh'), diff --git a/rainbow/build/image/image_builder.py b/rainbow/build/image_builder.py similarity index 64% rename from rainbow/build/image/image_builder.py rename to rainbow/build/image_builder.py index e716b9d..a56a22e 100644 --- a/rainbow/build/image/image_builder.py +++ b/rainbow/build/image_builder.py @@ -18,24 +18,23 @@ import os import shutil +import subprocess import tempfile -import docker - class ImageBuilder: """ Builds an image from source code """ + __NO_CACHE = 'no_cache' + def __init__(self, config, base_path, relative_source_path, tag): """ - TODO: pydoc - - :param config: - :param base_path: - :param relative_source_path: - :param tag: + :param config: task/service config + :param base_path: directory containing rainbow yml + :param relative_source_path: source path relative to rainbow yml + :param tag: image tag """ self.base_path = base_path self.relative_source_path = relative_source_path @@ -51,27 +50,44 @@ class ImageBuilder: temp_dir = self.__temp_dir() self.__copy_source_code(temp_dir) - self.__write_additional_files(temp_dir) - - # TODO: log docker output - docker_client = docker.from_env() - docker_client.images.build(path=temp_dir, tag=self.tag) - docker_client.close() + self._write_additional_files(temp_dir) + + no_cache = '' + if self.__NO_CACHE in self.config and self.config[self.__NO_CACHE]: + no_cache = '--no-cache=true' + + docker_build_command = f'docker build {no_cache} --progress=plain ' + \ + f'--tag {self.tag} {self._build_flags()} {temp_dir}' + + if self._use_buildkit(): + docker_build_command = f'DOCKER_BUILDKIT=1 {docker_build_command}' + + print(docker_build_command) + + docker_build_out = '' + try: + docker_build_out = subprocess.check_output(docker_build_command, + shell=True, stderr=subprocess.STDOUT, + timeout=240) + except subprocess.CalledProcessError as e: + docker_build_out = e.output + raise e + finally: + print('=' * 80) + for line in str(docker_build_out)[2:-3].split('\\n'): + print(line) + print('=' * 80) self.__remove_dir(temp_dir) print(f'[X] Building image: {self.tag} (Success).') + return docker_build_out + def __copy_source_code(self, temp_dir): self.__copy_dir(os.path.join(self.base_path, self.relative_source_path), temp_dir) - def __write_additional_files(self, temp_dir): - # TODO: move requirements.txt related code to a parent class for python image builders. - requirements_file_path = os.path.join(temp_dir, 'requirements.txt') - if not os.path.exists(requirements_file_path): - with open(requirements_file_path, 'w'): - pass - + def _write_additional_files(self, temp_dir): for file in [self._dockerfile_path()] + self._additional_files_from_paths(): self.__copy_file(file, temp_dir) @@ -117,6 +133,18 @@ class ImageBuilder: """ return [] + def _build_flags(self): + """ + Additional build flags to add to docker build command. + """ + return '' + + def _use_buildkit(self): + """ + overwrite with True to use docker buildkit + """ + return False + class ServiceImageBuilderMixin(object): pass diff --git a/rainbow/build/python.py b/rainbow/build/python.py new file mode 100644 index 0000000..0961d2b --- /dev/null +++ b/rainbow/build/python.py @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +from rainbow.build.image_builder import ImageBuilder + + +class BasePythonImageBuilder(ImageBuilder): + """ + Base class for building python images. + """ + + __PIP_CONF = 'pip_conf' + + def __init__(self, config, base_path, relative_source_path, tag): + super().__init__(config, base_path, relative_source_path, tag) + + @staticmethod + def _dockerfile_path(): + raise NotImplementedError() + + def _write_additional_files(self, temp_dir): + requirements_file_path = os.path.join(temp_dir, 'requirements.txt') + if not os.path.exists(requirements_file_path): + with open(requirements_file_path, 'w'): + pass + + super()._write_additional_files(temp_dir) + + def _additional_files_from_filename_content_pairs(self): + with open(self._dockerfile_path()) as original: + data = original.read() + + data = self.__mount_pip_conf(data) + + return [('Dockerfile', data)] + + def __mount_pip_conf(self, data): + new_data = data + + if self.__PIP_CONF in self.config: + new_data = '# syntax = docker/dockerfile:1.0-experimental\n' + data + new_data = new_data.replace('{{mount}}', + '--mount=type=secret,id=pip_config,dst=/etc/pip.conf \\\n') + else: + new_data = new_data.replace('{{mount}} ', '') + + return new_data + + def _build_flags(self): + if self.__PIP_CONF in self.config: + return f'--secret id=pip_config,src={self.config[self.__PIP_CONF]}' + else: + return '' + + def _use_buildkit(self): + if self.__PIP_CONF in self.config: + return True diff --git a/rainbow/build/service/python_server/Dockerfile b/rainbow/build/service/python_server/Dockerfile index 6119437..4d4254f 100644 --- a/rainbow/build/service/python_server/Dockerfile +++ b/rainbow/build/service/python_server/Dockerfile @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Use an official Python runtime as a parent image FROM python:3.7-slim @@ -11,14 +29,14 @@ WORKDIR /app # Be careful when changing this code. ! # Install any needed packages specified in python_server_requirements.txt and requirements.txt -COPY ./python_server_requirements.txt /app +COPY ./python_server_requirements.txt /app/ RUN pip install -r python_server_requirements.txt -COPY ./requirements.txt /app -RUN pip install -r requirements.txt +COPY ./requirements.txt /app/ +RUN {{mount}} pip install -r requirements.txt # Copy the current directory contents into the container at /app RUN echo "Copying source code.." -COPY . /app +COPY . /app/ CMD python -u rainbow_python_server.py diff --git a/rainbow/build/service/python_server/python_server.py b/rainbow/build/service/python_server/python_server.py index 3404abf..0b2537d 100644 --- a/rainbow/build/service/python_server/python_server.py +++ b/rainbow/build/service/python_server/python_server.py @@ -20,10 +20,11 @@ import os import yaml -from rainbow.build.image.image_builder import ImageBuilder, ServiceImageBuilderMixin +from rainbow.build.image_builder import ServiceImageBuilderMixin +from rainbow.build.python import BasePythonImageBuilder -class PythonServerImageBuilder(ImageBuilder, ServiceImageBuilderMixin): +class PythonServerImageBuilder(BasePythonImageBuilder, ServiceImageBuilderMixin): def __init__(self, config, base_path, relative_source_path, tag): super().__init__(config, base_path, relative_source_path, tag) @@ -40,4 +41,5 @@ class PythonServerImageBuilder(ImageBuilder, ServiceImageBuilderMixin): ] def _additional_files_from_filename_content_pairs(self): - return [('service.yml', yaml.safe_dump(self.config))] + return super()._additional_files_from_filename_content_pairs() + \ + [('service.yml', yaml.safe_dump(self.config))] diff --git a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py b/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py index eb6fa83..c44e80b 100644 --- a/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py +++ b/rainbow/runners/airflow/operators/kubernetes_pod_operator_with_input_output.py @@ -67,7 +67,6 @@ class PrepareInputOperator(KubernetesPodOperator): else: raise ValueError(f'Unknown config type: {self.input_type}') - # TODO: pass run_id as well as env var run_id = context['dag_run'].run_id print(f'run_id = {run_id}') @@ -145,4 +144,8 @@ class KubernetesPodOperatorWithInputAndOutput(KubernetesPodOperator): self.log.info(f'Empty input for task {self.task_split}.') + run_id = context['dag_run'].run_id + print(f'run_id = {run_id}') + + self.env_vars.update({'run_id': run_id}) return super().execute(context) diff --git a/run_tests.sh b/run_tests.sh index 3e5cd2f..8fdae7a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,3 +1,5 @@ #!/bin/sh -python -m unittest \ No newline at end of file +export TMPDIR=/tmp + +python -m unittest diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py index 63fc8fa..ecdaced 100644 --- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py +++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + import os import threading import time @@ -41,23 +42,47 @@ class TestPythonServer(TestCase): self.docker_client.close() def test_build_python_server(self): + build_out = self.__test_build_python_server() + + self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command') + + def test_build_python_server_with_pip_conf(self): + build_out = self.__test_build_python_server(use_pip_conf=True) + + self.assertTrue( + 'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip insta...' in build_out, + 'Incorrect pip command') + + def __test_build_python_server(self, use_pip_conf=False): base_path = os.path.join(os.path.dirname(__file__), '../../../rainbow') - builder = PythonServerImageBuilder(config=self.config, + + config = self.__create_conf('my_task') + + if use_pip_conf: + config['pip_conf'] = os.path.join(base_path, 'pip.conf') + + builder = PythonServerImageBuilder(config=config, base_path=base_path, relative_source_path='myserver', tag=self.image_name) - builder.build() + build_out = str(builder.build()) thread = threading.Thread(target=self.__run_container, args=[self.image_name]) thread.daemon = True thread.start() - time.sleep(2) + time.sleep(5) + + print('Sending request to server') + + server_response = str(urllib.request.urlopen('http://localhost:9294/myendpoint1').read()) + + print(f'Response from server: {server_response}') - server_response = urllib.request.urlopen("http://localhost:9294/myendpoint1";).read() + self.assertEqual("b'1'", server_response) - self.assertEqual("b'1'", str(server_response)) + return build_out def __remove_containers(self): print(f'Stopping containers with image: {self.image_name}') @@ -92,6 +117,7 @@ class TestPythonServer(TestCase): 'input_type': 'my_input_type', 'input_path': 'my_input', 'output_path': '/my_output.json', + 'no_cache': True, 'endpoints': [ { 'endpoint': '/myendpoint1', diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py index 7376987..81b5cc3 100644 --- a/tests/runners/airflow/build/python/test_python_image_builder.py +++ b/tests/runners/airflow/build/python/test_python_image_builder.py @@ -16,6 +16,8 @@ # specific language governing permissions and limitations # under the License. import os +import shutil +import tempfile from unittest import TestCase import docker @@ -24,46 +26,106 @@ from rainbow.build.image.python.python import PythonImageBuilder class TestPythonImageBuilder(TestCase): + __IMAGE_NAME = 'rainbow_image' + __OUTPUT_PATH = '/mnt/vol1/my_output.json' + + def setUp(self) -> None: + super().setUp() + os.environ['TMPDIR'] = '/tmp' + self.temp_dir = self.__temp_dir() + self.temp_airflow_dir = self.__temp_dir() + + def tearDown(self) -> None: + super().tearDown() + self.__remove_dir(self.temp_dir) + self.__remove_dir(self.temp_airflow_dir) def test_build(self): - config = self.__create_conf('my_task') + build_out = self.__test_build() + + self.assertTrue('RUN pip install -r requirements.txt' in build_out, 'Incorrect pip command') + + self.__test_image() - image_name = config['image'] + def test_build_with_pip_conf(self): + build_out = self.__test_build(use_pip_conf=True) + + self.assertTrue( + 'RUN --mount=type=secret,id=pip_config,dst=/etc/pip.conf pip insta...' in build_out, + 'Incorrect pip command') + + self.__test_image() + + def __test_build(self, use_pip_conf=False): + config = self.__create_conf('my_task') base_path = os.path.join(os.path.dirname(__file__), '../../rainbow') + if use_pip_conf: + config['pip_conf'] = os.path.join(base_path, 'pip.conf') + builder = PythonImageBuilder(config=config, base_path=base_path, relative_source_path='helloworld', - tag=image_name) + tag=self.__IMAGE_NAME) - builder.build() + build_out = str(builder.build()) - # TODO: elaborate test of image, validate input/output + return build_out + def __test_image(self): docker_client = docker.from_env() - docker_client.images.get(image_name) + docker_client.images.get(self.__IMAGE_NAME) - cmd = 'export RAINBOW_INPUT="{}" && ' + \ + cmd = 'export RAINBOW_INPUT="{\\"x\\": 1}" && ' + \ 'sh container-setup.sh && ' + \ 'python hello_world.py && ' + \ - 'sh container-teardown.sh' + f'sh container-teardown.sh {self.__OUTPUT_PATH}' cmds = ['/bin/bash', '-c', cmd] - container_log = docker_client.containers.run(image_name, cmds) + container_log = docker_client.containers.run(self.__IMAGE_NAME, + cmds, + volumes={ + self.temp_dir: { + 'bind': '/mnt/vol1', + 'mode': 'rw' + }, + self.temp_airflow_dir: { + 'bind': '/airflow/xcom', + 'mode': 'rw'}, + }) docker_client.close() - self.assertEqual("b'Hello world!\\n\\n{}\\n'", str(container_log)) + print(container_log) - @staticmethod - def __create_conf(task_id): + self.assertEqual( + "b\"Writing rainbow input..\\n" + + "Hello world!\\n\\n" + + "rainbow_input.json contents = {'x': 1}\\n" + + "Writing rainbow output..\\n\"", + str(container_log)) + + with open(os.path.join(self.temp_airflow_dir, 'return.json')) as file: + self.assertEqual(file.read(), '{"a": 1, "b": 2}') + + def __create_conf(self, task_id): return { 'task': task_id, 'cmd': 'foo bar', - 'image': 'rainbow_image', + 'image': self.__IMAGE_NAME, 'source': 'baz', 'input_type': 'my_input_type', 'input_path': 'my_input', - 'output_path': '/my_output.json' + 'no_cache': True, + 'output_path': self.__OUTPUT_PATH, } + + @staticmethod + def __temp_dir(): + temp_dir = tempfile.mkdtemp() + return temp_dir + + @staticmethod + def __remove_dir(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) diff --git a/tests/runners/airflow/build/test_build_rainbows.py b/tests/runners/airflow/build/test_build_rainbows.py index c5d8ea7..7e01245 100644 --- a/tests/runners/airflow/build/test_build_rainbows.py +++ b/tests/runners/airflow/build/test_build_rainbows.py @@ -42,7 +42,7 @@ class TestBuildRainbows(TestCase): def __remove_images(self): for image_name in self.__image_names: if len(self.docker_client.images.list(image_name)) > 0: - self.docker_client.images.remove(image=image_name) + self.docker_client.images.remove(image=image_name, force=True) def test_build_rainbow(self): build_rainbows.build_rainbows(os.path.join(os.path.dirname(__file__), '../rainbow')) diff --git a/tests/runners/airflow/rainbow/helloworld/hello_world.py b/tests/runners/airflow/rainbow/helloworld/hello_world.py index 3eae465..95f4e73 100644 --- a/tests/runners/airflow/rainbow/helloworld/hello_world.py +++ b/tests/runners/airflow/rainbow/helloworld/hello_world.py @@ -16,12 +16,14 @@ # specific language governing permissions and limitations # under the License. import json +import os -print('Hello world!') -print() +print('Hello world!\n') with open('/rainbow_input.json') as file: - print(json.loads(file.readline())) + print(f'rainbow_input.json contents = {json.loads(file.readline())}') -with open('/output.json', 'w') as file: +os.makedirs('/mnt/vol1/', exist_ok=True) + +with open('/mnt/vol1/my_output.json', 'w') as file: file.write(json.dumps({'a': 1, 'b': 2})) diff --git a/tests/runners/airflow/rainbow/helloworld/hello_world.py b/tests/runners/airflow/rainbow/pip.conf similarity index 78% copy from tests/runners/airflow/rainbow/helloworld/hello_world.py copy to tests/runners/airflow/rainbow/pip.conf index 3eae465..217e5db 100644 --- a/tests/runners/airflow/rainbow/helloworld/hello_world.py +++ b/tests/runners/airflow/rainbow/pip.conf @@ -15,13 +15,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json - -print('Hello world!') -print() - -with open('/rainbow_input.json') as file: - print(json.loads(file.readline())) - -with open('/output.json', 'w') as file: - file.write(json.dumps({'a': 1, 'b': 2})) diff --git a/tests/runners/airflow/rainbow/rainbow.yml b/tests/runners/airflow/rainbow/rainbow.yml index 0b08a1f..77af37b 100644 --- a/tests/runners/airflow/rainbow/rainbow.yml +++ b/tests/runners/airflow/rainbow/rainbow.yml @@ -29,8 +29,8 @@ pipelines: key1: val1 key2: val2 metrics: - namespace: TestNamespace - backends: [ 'cloudwatch' ] + namespace: TestNamespace + backends: [ 'cloudwatch' ] tasks: - task: my_static_input_task type: python @@ -42,7 +42,7 @@ pipelines: env2: "b" input_type: static input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]' - output_path: /output.json + output_path: /mnt/vol1/my_output.json cmd: python -u hello_world.py - task: my_parallelized_static_input_task type: python @@ -55,7 +55,7 @@ pipelines: input_path: '[ { "foo": "bar" }, { "foo": "baz" } ]' split_input: True executors: 2 - cmd: python -u helloworld.py + cmd: python -u hello_world.py - task: my_task_output_input_task type: python description: task with input from other task's output
