This is an automated email from the ASF dual-hosted git repository. aviemzur pushed a commit to branch revert-19-logging in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 43cc09f830e8794a69b38112b669742da506f2f0 Author: Aviem Zur <[email protected]> AuthorDate: Wed Jan 6 09:56:02 2021 +0200 Revert "[LIMINAL-5] change prints to log" This reverts commit 4e323809042800ca296bc65e04baa23b1dfe6b71. --- .../helloworld/hello_world.py | 8 +++- liminal/build/image_builder.py | 13 +++--- liminal/build/liminal_apps_builder.py | 21 +++++----- liminal/core/environment.py | 8 ++-- liminal/core/util/files_util.py | 5 +-- liminal/kubernetes/volume_util.py | 2 +- liminal/logging/__init__.py | 17 -------- liminal/logging/logging_setup.py | 47 ---------------------- liminal/runners/airflow/dag/liminal_dags.py | 13 +++--- scripts/liminal | 20 ++++----- setup.py | 4 +- .../python/test_python_server_image_builder.py | 18 ++++----- .../build/python/test_python_image_builder.py | 5 +-- .../airflow/build/test_liminal_apps_builder.py | 1 + .../airflow/liminal/write_outputs/write_outputs.py | 5 +-- tests/runners/airflow/tasks/test_python.py | 3 +- 16 files changed, 59 insertions(+), 131 deletions(-) diff --git a/examples/liminal-getting-started/helloworld/hello_world.py b/examples/liminal-getting-started/helloworld/hello_world.py index a26752c..78fbd9c 100644 --- a/examples/liminal-getting-started/helloworld/hello_world.py +++ b/examples/liminal-getting-started/helloworld/hello_world.py @@ -16,8 +16,14 @@ # specific language governing permissions and limitations # under the License. import json +import os print('Hello world!\n') +# with open('/mnt/gettingstartedvol/hello_world.json') as file: +# print(f'hello_world.json contents = {json.loads(file.readline())}') +# +# os.makedirs('/mnt/vol1/', exist_ok=True) + with open('/mnt/gettingstartedvol/hello_world_output.json', 'w') as file: - file.write(json.dumps({'hello': 1, 'world': 2})) + file.write(json.dumps({'hello': 1, 'world': 2})) \ No newline at end of file diff --git a/liminal/build/image_builder.py b/liminal/build/image_builder.py index 11ff9f0..a7ca13b 100644 --- a/liminal/build/image_builder.py +++ b/liminal/build/image_builder.py @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. -import logging import os import shutil import subprocess @@ -46,7 +45,7 @@ class ImageBuilder: """ Builds source code into an image. """ - logging.info(f'[ ] Building image: {self.tag}') + print(f'[ ] Building image: {self.tag}') temp_dir = self.__temp_dir() @@ -68,7 +67,7 @@ class ImageBuilder: if self._use_buildkit(): docker_build_command = f'DOCKER_BUILDKIT=1 {docker_build_command}' - logging.info(docker_build_command) + print(docker_build_command) docker_build_out = '' try: @@ -79,14 +78,14 @@ class ImageBuilder: docker_build_out = e.output raise e finally: - logging.info('=' * 80) + print('=' * 80) for line in str(docker_build_out)[2:-3].split('\\n'): - logging.info(line) - logging.info('=' * 80) + print(line) + print('=' * 80) self.__remove_dir(temp_dir) - logging.info(f'[X] Building image: {self.tag} (Success).') + print(f'[X] Building image: {self.tag} (Success).') return docker_build_out diff --git a/liminal/build/liminal_apps_builder.py b/liminal/build/liminal_apps_builder.py index 19b065f..dd98477 100644 --- a/liminal/build/liminal_apps_builder.py +++ b/liminal/build/liminal_apps_builder.py @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. -import logging import os import yaml @@ -32,7 +31,7 @@ def build_liminal_apps(path): config_files = files_util.find_config_files(path) for config_file in config_files: - logging.info(f'Building artifacts for file: {config_file}') + print(f'Building artifacts for file: {config_file}') base_path = os.path.dirname(config_file) @@ -52,8 +51,7 @@ def build_liminal_apps(path): else: raise ValueError(f'No such task type: {task_type}') else: - logging.info( - f'No source configured for task {task_name}, skipping build..') + print(f'No source configured for task {task_name}, skipping build..') if 'services' in liminal_config: for service in liminal_config['services']: @@ -74,7 +72,7 @@ def __build_image(base_path, builder_config, builder): tag=builder_config['image']) builder_instance.build() else: - logging.info(f"No source provided for {builder_config['name']}, skipping.") + print(f"No source provided for {builder_config['name']}, skipping.") def __get_task_build_class(task_type): @@ -85,13 +83,13 @@ def __get_service_build_class(service_type): return service_build_types.get(service_type, None) -logging.info(f'Loading image builder implementations..') +print(f'Loading image builder implementations..') # TODO: add configuration for user image builders package image_builders_package = 'liminal.build.image' # user_image_builders_package = 'TODO: user_image_builders_package' -TASK_BUILD_CLASSES = class_util.find_subclasses_in_packages( +task_build_classes = class_util.find_subclasses_in_packages( [image_builders_package], ImageBuilder) @@ -101,10 +99,11 @@ def get_types_dict(task_build_classes): return {x.split(".")[-2]: c for x, c in task_build_classes.items()} -task_build_types = get_types_dict(TASK_BUILD_CLASSES) +task_build_types = get_types_dict(task_build_classes) -logging.info(f'Finished loading image builder implementations: {TASK_BUILD_CLASSES}') -logging.info(f'Loading service image builder implementations..') +print(f'Finished loading image builder implementations: {task_build_classes}') + +print(f'Loading service image builder implementations..') # TODO: add configuration for user service image builders package service_builders_package = 'liminal.build.service' @@ -115,4 +114,4 @@ service_build_classes = class_util.find_subclasses_in_packages( ServiceImageBuilderMixin) service_build_types = get_types_dict(service_build_classes) -logging.info(f'Finished loading service image builder implementations: {service_build_classes}') +print(f'Finished loading service image builder implementations: {service_build_classes}') diff --git a/liminal/core/environment.py b/liminal/core/environment.py index acea4e4..42e14a4 100644 --- a/liminal/core/environment.py +++ b/liminal/core/environment.py @@ -18,7 +18,7 @@ import os import subprocess -import logging +from pathlib import Path DEFAULT_DAGS_ZIP_NAME = 'liminal.zip' DEFAULT_LIMINAL_HOME = os.path.expanduser('~/liminal_home') @@ -29,8 +29,8 @@ LIMINAL_VERSION_PARAM_NAME = 'LIMINAL_VERSION' def get_liminal_home(): if not os.environ.get(LIMINAL_HOME_PARAM_NAME): - logging.info("no environment parameter called LIMINAL_HOME detected") - logging.info(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory") + print("no environment parameter called LIMINAL_HOME detected") + print(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory") os.environ[LIMINAL_HOME_PARAM_NAME] = DEFAULT_LIMINAL_HOME return os.environ.get(LIMINAL_HOME_PARAM_NAME, DEFAULT_LIMINAL_HOME) @@ -58,6 +58,6 @@ def get_liminal_version(): value = pip_res[pip_res.index(' @ ') + 3:] else: value = pip_res - logging.info(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}') + print(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}') os.environ[LIMINAL_VERSION_PARAM_NAME] = value return os.environ.get(LIMINAL_VERSION_PARAM_NAME, 'apache-liminal') diff --git a/liminal/core/util/files_util.py b/liminal/core/util/files_util.py index 48ab9b7..e611005 100644 --- a/liminal/core/util/files_util.py +++ b/liminal/core/util/files_util.py @@ -17,15 +17,14 @@ # under the License. import os -import logging def find_config_files(path): files = [] - logging.info(path) + print(path) for r, d, f in os.walk(path): for file in f: if os.path.basename(file) in ['liminal.yml', 'liminal.yaml']: - logging.info(os.path.join(r, file)) + print(os.path.join(r, file)) files.append(os.path.join(r, file)) return files diff --git a/liminal/kubernetes/volume_util.py b/liminal/kubernetes/volume_util.py index eff1fd0..259faa8 100644 --- a/liminal/kubernetes/volume_util.py +++ b/liminal/kubernetes/volume_util.py @@ -34,7 +34,7 @@ def create_local_volumes(liminal_config, base_dir): for volume_config in volumes_config: if 'local' in volume_config: - logging.info(f'Creating local kubernetes volume if needed: {volume_config}') + print(f'Creating local kubernetes volume if needed: {volume_config}') path = volume_config['local']['path'] if path.startswith(".."): path = os.path.join(base_dir, path) diff --git a/liminal/logging/__init__.py b/liminal/logging/__init__.py deleted file mode 100644 index 217e5db..0000000 --- a/liminal/logging/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/liminal/logging/logging_setup.py b/liminal/logging/logging_setup.py deleted file mode 100644 index 995fa02..0000000 --- a/liminal/logging/logging_setup.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -from logging.handlers import RotatingFileHandler - -from liminal.core import environment - -LIMINAL = 'liminal' -LOGS_DIR = 'logs' -MAX_FILE_SIZE = 10485760 # 10 MB - - -def logging_initialization(): - root_logger = logging.getLogger() - - log_formatter = logging.Formatter( - '[%(asctime)s] [%(filename)s:%(lineno)d] %(levelname)s - %(message)s', - '%m-%d %H:%M:%S' - ) - - file_handler = RotatingFileHandler( - f'{environment.get_liminal_home()}/{LOGS_DIR}/{LIMINAL}.log', - maxBytes=MAX_FILE_SIZE, - backupCount=3 - ) - root_logger.addHandler(file_handler) - root_logger.setLevel(logging.INFO) - - [h.setFormatter(log_formatter) for h in root_logger.handlers] - - logging.info('Logging initialization completed') diff --git a/liminal/runners/airflow/dag/liminal_dags.py b/liminal/runners/airflow/dag/liminal_dags.py index 03911b1..3e5e1a3 100644 --- a/liminal/runners/airflow/dag/liminal_dags.py +++ b/liminal/runners/airflow/dag/liminal_dags.py @@ -27,7 +27,6 @@ from liminal.core.util import files_util from liminal.runners.airflow.model.task import Task from liminal.runners.airflow.tasks.defaults.job_end import JobEndTask from liminal.runners.airflow.tasks.defaults.job_start import JobStartTask -import logging __DEPENDS_ON_PAST = 'depends_on_past' @@ -36,13 +35,13 @@ def register_dags(configs_path): """ Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs. """ - logging.info(f'Registering DAG from path: {configs_path}') + print(f'Registering DAG from path: {configs_path}') config_files = files_util.find_config_files(configs_path) dags = [] - logging.info(f'found {len(config_files)} in path: {configs_path}') + print(f'found {len(config_files)} in path: {configs_path}') for config_file in config_files: - logging.info(f'Registering DAG for file: {config_file}') + print(f'Registering DAG for file: {config_file}') with open(config_file) as stream: config = yaml.safe_load(stream) @@ -85,7 +84,7 @@ def register_dags(configs_path): job_end_task = JobEndTask(dag, config, pipeline, {}, parent, 'all_done') job_end_task.apply_task_to_dag() - logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}') + print(f'registered DAG {dag.dag_id}: {dag.tasks}') globals()[pipeline_name] = dag dags.append(dag) @@ -93,7 +92,7 @@ def register_dags(configs_path): return dags -logging.info(f'Loading task implementations..') +print(f'Loading task implementations..') # TODO: add configuration for user tasks package impl_packages = 'liminal.runners.airflow.tasks' @@ -109,7 +108,7 @@ tasks_by_liminal_name = tasks_by_liminal_name( class_util.find_subclasses_in_packages([impl_packages], Task) ) -logging.info(f'Finished loading task implementations: {tasks_by_liminal_name}') +print(f'Finished loading task implementations: {tasks_by_liminal_name}') def get_task_class(task_type): diff --git a/scripts/liminal b/scripts/liminal index ff4767e..3c7b132 100755 --- a/scripts/liminal +++ b/scripts/liminal @@ -18,7 +18,6 @@ # specific language governing permissions and limitations # under the License. -import logging import os import pathlib import shutil @@ -28,16 +27,13 @@ import sys import click import yaml -import scripts +import scripts as s from liminal.build import liminal_apps_builder from liminal.core import environment from liminal.core.util import files_util from liminal.kubernetes import volume_util -from liminal.logging.logging_setup import logging_initialization from liminal.runners.airflow import dag -logging_initialization() - try: import importlib.resources as pkg_resources except ImportError: @@ -68,7 +64,6 @@ def build(path): def deploy_liminal_core_internal(clean): - # noinspection PyTypeChecker with pkg_resources.path(dag, 'liminal_dags.py') as p: dags_path = p os.makedirs(environment.get_dags_dir(), exist_ok=True) @@ -77,13 +72,13 @@ def deploy_liminal_core_internal(clean): # initialize the env. variable which indicates to the docke compose which # liminal to install in airflow docker liminal_version = environment.get_liminal_version() - logging.info(f'Deploying liminal version: {liminal_version}') + print(f'Deploying liminal version: {liminal_version}') # if liminal is installed from local file - the developer needs to put it in the /scripts folder # in which case it will end up inside the container during build if liminal_version.find("file://") > -1: local_file_name = os.path.basename(liminal_version) full_path = os.path.join('/opt/airflow/dags', local_file_name) - logging.info( + print( f'Liminal was installed locally, changing the LIMINAL_VERSION parameter to {full_path}') os.environ[environment.LIMINAL_VERSION_PARAM_NAME] = full_path if clean: @@ -103,7 +98,7 @@ def docker_compose_command(command_name, args): '-p liminal --project-directory ' f'{project_dir} {command_name} {concated_args}' ] - logging.info(run_command[0]) + print(run_command[0]) if 'follow' in str(args): subprocess.call(run_command, env=os.environ, shell=True) return '', '' @@ -159,7 +154,7 @@ def logs(follow, tail): docker_compose_command('logs', ['--follow']) if tail > 0: stdout, stderr = docker_compose_command('logs', [f'--tail={tail}']) - logging.info(stdout) + print(stdout) @cli.command("start", @@ -167,7 +162,7 @@ def logs(follow, tail): "Make sure docker is running on your machine") def start(): liminal_version = environment.get_liminal_version() - logging.info(f'starting liminal version: {liminal_version}') + print(f'starting liminal version: {liminal_version}') if docker_is_running(): # initialize liminal home by default environment.get_liminal_home() @@ -175,8 +170,7 @@ def start(): def get_docker_compose_paths(): - # noinspection PyTypeChecker - with pkg_resources.path(scripts, 'docker-compose.yml') as p: + with pkg_resources.path(s, 'docker-compose.yml') as p: docker_compose_path = p project_dir = pathlib.Path(docker_compose_path).parent.parent.absolute() return docker_compose_path, project_dir diff --git a/setup.py b/setup.py index 6bc9845..b06fd17 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import logging import os import setuptools @@ -27,7 +26,8 @@ with open("README.md", "r") as fh: with open('requirements.txt') as f: requirements = f.read().splitlines() - logging.info(requirements) + print(requirements) + setuptools.setup( name="apache-liminal", diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py index 88f9663..18215d3 100644 --- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py +++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py @@ -17,7 +17,6 @@ # under the License. import json -import logging import os import threading import time @@ -27,9 +26,8 @@ from unittest import TestCase import docker -from liminal.build.python import PythonImageVersions from liminal.build.service.python_server.python_server import PythonServerImageBuilder - +from liminal.build.python import PythonImageVersions class TestPythonServer(TestCase): @@ -83,7 +81,7 @@ class TestPythonServer(TestCase): time.sleep(5) - logging.info('Sending request to server') + print('Sending request to server') json_string = '{"key1": "val1", "key2": "val2"}' @@ -94,33 +92,33 @@ class TestPythonServer(TestCase): data=json_string.encode(encoding) ).read().decode(encoding)) - logging.info(f'Response from server: {server_response}') + print(f'Response from server: {server_response}') self.assertEqual(f'Input was: {json.loads(json_string)}', server_response) return build_out def __remove_containers(self): - logging.info(f'Stopping containers with image: {self.image_name}') + print(f'Stopping containers with image: {self.image_name}') all_containers = self.docker_client.containers matching_containers = all_containers.list(filters={'ancestor': self.image_name}) for container in matching_containers: container_id = container.id - logging.info(f'Stopping container {container_id}') + print(f'Stopping container {container_id}') self.docker_client.api.stop(container_id) - logging.info(f'Removing container {container_id}') + print(f'Removing container {container_id}') self.docker_client.api.remove_container(container_id) self.docker_client.containers.prune() def __run_container(self, image_name): try: - logging.info(f'Running container for image: {image_name}') + print(f'Running container for image: {image_name}') self.docker_client.containers.run(image_name, ports={'80/tcp': 9294}) except Exception as err: - logging.exception(err) + print(err) pass @staticmethod diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py index 7c0032a..56667bd 100644 --- a/tests/runners/airflow/build/python/test_python_image_builder.py +++ b/tests/runners/airflow/build/python/test_python_image_builder.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import logging import os import shutil import tempfile @@ -42,7 +41,7 @@ class TestPythonImageBuilder(TestCase): self.__remove_dir(self.temp_airflow_dir) def test_build(self): - for python_version in [None, + for python_version in [None , PythonImageVersions().supported_versions[0]]: build_out = self.__test_build(python_version=python_version) self.assertTrue('RUN pip install -r requirements.txt' in build_out, @@ -104,7 +103,7 @@ class TestPythonImageBuilder(TestCase): docker_client.close() - logging.info(container_log) + print(container_log) self.assertEqual( "b'" diff --git a/tests/runners/airflow/build/test_liminal_apps_builder.py b/tests/runners/airflow/build/test_liminal_apps_builder.py index c011a5c..d8407e4 100644 --- a/tests/runners/airflow/build/test_liminal_apps_builder.py +++ b/tests/runners/airflow/build/test_liminal_apps_builder.py @@ -23,6 +23,7 @@ import docker from liminal.build import liminal_apps_builder + class TestLiminalAppsBuilder(TestCase): __image_names = [ 'my_python_task_img', diff --git a/tests/runners/airflow/liminal/write_outputs/write_outputs.py b/tests/runners/airflow/liminal/write_outputs/write_outputs.py index 4452187..806ce0e 100644 --- a/tests/runners/airflow/liminal/write_outputs/write_outputs.py +++ b/tests/runners/airflow/liminal/write_outputs/write_outputs.py @@ -17,7 +17,6 @@ # under the License. import json -import logging import os split_id = int(os.environ['LIMINAL_SPLIT_ID']) @@ -29,7 +28,7 @@ outputs_dir = '/mnt/vol1/outputs/' if not os.path.exists(outputs_dir): os.makedirs(outputs_dir) -logging.info(f'Running write_outputs for split id {split_id} [NUM_SPLITS = {num_splits}]') +print(f'Running write_outputs for split id {split_id} [NUM_SPLITS = {num_splits}]') for filename in os.listdir(inputs_dir): with open(os.path.join(inputs_dir, filename)) as infile, \ @@ -37,6 +36,6 @@ for filename in os.listdir(inputs_dir): outputs_dir, filename.replace('input', 'output').replace('.json', '.txt') ), 'w') as outfile: - logging.info(f'Writing output file: {outfile.name}') + print(f'Writing output file: {outfile.name}') data = json.loads(infile.read()) outfile.write(data['mykey']) diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py index 1a91973..418e2eb 100644 --- a/tests/runners/airflow/tasks/test_python.py +++ b/tests/runners/airflow/tasks/test_python.py @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import logging import os import tempfile import unittest @@ -59,7 +58,7 @@ class TestPythonTask(TestCase): task1.apply_task_to_dag() for task in dag.tasks: - logging.info(f'Executing task {task.task_id}') + print(f'Executing task {task.task_id}') task.execute({}) inputs_dir = os.path.join(self.temp_dir, 'inputs')
