Repository: incubator-airflow Updated Branches: refs/heads/master f7f585a89 -> d62a03767
[AIRFLOW-2539][AIRFLOW-2359] Move remaing log config to configuration file Closes #3435 from NielsZeilemaker/env_logging_filename Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d62a0376 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d62a0376 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d62a0376 Branch: refs/heads/master Commit: d62a03767193a6d056063354355a5cacf4c2c0ca Parents: f7f585a Author: niels <[email protected]> Authored: Fri Jun 15 13:25:26 2018 +0200 Committer: Fokko Driesprong <[email protected]> Committed: Fri Jun 15 13:25:26 2018 +0200 ---------------------------------------------------------------------- UPDATING.md | 13 +++++++++++++ airflow/config_templates/airflow_local_settings.py | 13 ++++++------- airflow/config_templates/default_airflow.cfg | 10 ++++++++++ airflow/config_templates/default_test.cfg | 8 ++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d62a0376/UPDATING.md ---------------------------------------------------------------------- diff --git a/UPDATING.md b/UPDATING.md index b32b2ac..1f2eb83 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -77,6 +77,19 @@ Header row will be added only if this parameter is set True and also in that cas With Airflow 1.9 or lower, there were two connection strings for the Google Cloud operators, both `google_cloud_storage_default` and `google_cloud_default`. This can be confusing and therefore the `google_cloud_storage_default` connection id has been replaced with `google_cloud_default` to make the connection id consistent across Airflow. +### Logging Configuration +With Airflow 1.9 or lower, `FILENAME_TEMPLATE`, `PROCESSOR_FILENAME_TEMPLATE`, `LOG_ID_TEMPLATE`, `END_OF_LOG_MARK` were configured in `airflow_local_settings.py`. These have been moved into the configuration file, and hence if you were using a custom configuration file the following defaults need to be added. +``` +[core] +fab_logging_level = WARN +log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log +log_processor_filename_template = {{{{ filename }}}}.log + +[elasticsearch] +elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} +elasticsearch_end_of_log_mark = end_of_log +``` + ## Airflow 1.9 ### SSH Hook updates, along with new SSH Operator & SFTP Operator http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d62a0376/airflow/config_templates/airflow_local_settings.py ---------------------------------------------------------------------- diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index c461ece..95150ab 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -30,7 +30,7 @@ LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper() # Flask appbuilder's info level log is very verbose, # so it's set to 'WARN' by default. -FAB_LOG_LEVEL = 'WARN' +FAB_LOG_LEVEL = conf.get('core', 'FAB_LOGGING_LEVEL').upper() LOG_FORMAT = conf.get('core', 'LOG_FORMAT') @@ -38,11 +38,8 @@ BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER') PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY') -FILENAME_TEMPLATE = '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log' - -PROCESSOR_FILENAME_TEMPLATE = '{{ filename }}.log' - -LOG_ID_TEMPLATE = '{dag_id}-{task_id}-{execution_date}-{try_number}' +FILENAME_TEMPLATE = conf.get('core', 'LOG_FILENAME_TEMPLATE') +PROCESSOR_FILENAME_TEMPLATE = conf.get('core', 'LOG_PROCESSOR_FILENAME_TEMPLATE') # Storage bucket url for remote logging # s3 buckets should start with "s3://" @@ -53,7 +50,9 @@ REMOTE_BASE_LOG_FOLDER = conf.get('core', 'REMOTE_BASE_LOG_FOLDER') ELASTICSEARCH_HOST = conf.get('elasticsearch', 'ELASTICSEARCH_HOST') -END_OF_LOG_MARK = 'end_of_log' +LOG_ID_TEMPLATE = conf.get('elasticsearch', 'ELASTICSEARCH_LOG_ID_TEMPLATE') + +END_OF_LOG_MARK = conf.get('elasticsearch', 'ELASTICSEARCH_END_OF_LOG_MARK') DEFAULT_LOGGING_CONFIG = { 'version': 1, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d62a0376/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 264757c..2f6130c 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -53,6 +53,7 @@ encrypt_s3_logs = False # Logging level logging_level = INFO +fab_logging_level = WARN # Logging class # Specify the class that will specify the logging configuration @@ -61,9 +62,15 @@ logging_level = INFO logging_config_class = # Log format +# we need to escape the curly braces by adding an additional curly brace log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s +# Log filename format +# we need to escape the curly braces by adding an additional curly brace +log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log +log_processor_filename_template = {{{{ filename }}}}.log + # Hostname by providing a path to a callable, which will resolve the hostname hostname_callable = socket:getfqdn @@ -542,6 +549,9 @@ hide_sensitive_variable_fields = True [elasticsearch] elasticsearch_host = +# we need to escape the curly braces by adding an additional curly brace +elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} +elasticsearch_end_of_log_mark = end_of_log [kubernetes] # The repository and tag of the Kubernetes Image for the Worker to Run http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d62a0376/airflow/config_templates/default_test.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg index 26851ef..cd4bd32 100644 --- a/airflow/config_templates/default_test.cfg +++ b/airflow/config_templates/default_test.cfg @@ -36,6 +36,9 @@ dags_folder = {TEST_DAGS_FOLDER} plugins_folder = {TEST_PLUGINS_FOLDER} base_log_folder = {AIRFLOW_HOME}/logs logging_level = INFO +fab_logging_level = WARN +log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log +log_processor_filename_template = {{{{ filename }}}}.log executor = SequentialExecutor sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/unittests.db load_examples = True @@ -114,3 +117,8 @@ max_tis_per_query = 512 [admin] hide_sensitive_variable_fields = True + +[elasticsearch] +elasticsearch_host = +elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} +elasticsearch_end_of_log_mark = end_of_log
