Repository: incubator-airflow Updated Branches: refs/heads/master e0491bcb2 -> 0d23d30d6
[AIRFLOW-1335] Use MemoryHandler for buffered logging Closes #2386 from saguziel/aguziel-buffer-logger- apache Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/0d23d30d Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/0d23d30d Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/0d23d30d Branch: refs/heads/master Commit: 0d23d30d6812fe1eb3cfb52d2992131cbf028062 Parents: e0491bc Author: Alex Guziel <[email protected]> Authored: Thu Jun 22 18:38:05 2017 -0700 Committer: Alex Guziel <[email protected]> Committed: Thu Jun 22 18:38:05 2017 -0700 ---------------------------------------------------------------------- airflow/bin/cli.py | 5 ++++- airflow/config_templates/default_airflow.cfg | 7 +++++++ airflow/config_templates/default_test.cfg | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 4b3a0ed..d4ddcb5 100755 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -90,7 +90,10 @@ def sigquit_handler(sig, frame): def setup_logging(filename): root = logging.getLogger() - handler = logging.FileHandler(filename) + handler_buffer_size = conf.getInt('core', 'logging_buffer_size') + handler_flush_level = conf.getInt('core', 'logging_flush_level') + # Buffered wrapper + handler = logging.MemoryHandler(handler_buffer_size, handler_flush_level, target=logging.FileHandler(filename)) formatter = logging.Formatter(settings.SIMPLE_LOG_FORMAT) handler.setFormatter(formatter) root.addHandler(handler) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index c6c1da2..13a4b20 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -49,6 +49,13 @@ s3_log_folder = # Logging level logging_level = INFO +# Size of logging buffer +logging_buffer_size = 4096 +# Logging level minimum to flush buffer before full +# See python doc on core module logging, DEBUG=10,INFO=20,WARN=30,ERROR=40 +# default to no buffering +logging_flush_level = 0 + # The executor class that airflow should use. Choices include # SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor executor = SequentialExecutor http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/config_templates/default_test.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg index 2fb5bb0..1c0d34c 100644 --- a/airflow/config_templates/default_test.cfg +++ b/airflow/config_templates/default_test.cfg @@ -29,6 +29,8 @@ dags_folder = {TEST_DAGS_FOLDER} plugins_folder = {TEST_PLUGINS_FOLDER} base_log_folder = {AIRFLOW_HOME}/logs logging_level = INFO +logging_flush_level = 0 +logging_buffer_size = 4096 executor = SequentialExecutor sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/unittests.db load_examples = True
