This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9b4eec2a0ea9b85a457e463cf2f251820e24ba0d Author: Ko HanJong <[email protected]> AuthorDate: Thu Jul 14 00:27:27 2022 +0900 Add %z for %(asctime)s to fix timezone for logs on UI (#24811) (cherry picked from commit 851e5cad165a043654116a8a2717c9d3643d8251) --- airflow/config_templates/airflow_local_settings.py | 11 ++++- airflow/config_templates/config.yml | 6 +++ airflow/config_templates/default_airflow.cfg | 1 + airflow/utils/log/timezone_aware.py | 49 ++++++++++++++++++++++ airflow/www/static/js/ti_log.js | 4 +- newsfragments/24811.significant.rst | 22 ++++++++++ 6 files changed, 90 insertions(+), 3 deletions(-) diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index 6684fd18e5..ea8a19e80c 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -38,6 +38,10 @@ FAB_LOG_LEVEL: str = conf.get_mandatory_value('logging', 'FAB_LOGGING_LEVEL').up LOG_FORMAT: str = conf.get_mandatory_value('logging', 'LOG_FORMAT') +LOG_FORMATTER_CLASS: str = conf.get_mandatory_value( + 'logging', 'LOG_FORMATTER_CLASS', fallback='airflow.utils.log.timezone_aware.TimezoneAware' +) + COLORED_LOG_FORMAT: str = conf.get_mandatory_value('logging', 'COLORED_LOG_FORMAT') COLORED_LOG: bool = conf.getboolean('logging', 'COLORED_CONSOLE_LOG') @@ -60,10 +64,13 @@ DEFAULT_LOGGING_CONFIG: Dict[str, Any] = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'airflow': {'format': LOG_FORMAT}, + 'airflow': { + 'format': LOG_FORMAT, + 'class': LOG_FORMATTER_CLASS, + }, 'airflow_coloured': { 'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT, - 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter', + 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else LOG_FORMATTER_CLASS, }, }, 'filters': { diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index f7409373c2..4ea090bcd9 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -625,6 +625,12 @@ type: string example: ~ default: "%%(asctime)s %%(levelname)s - %%(message)s" + - name: log_formatter_class + description: ~ + version_added: 2.3.4 + type: string + example: ~ + default: "airflow.utils.log.timezone_aware.TimezoneAware" - name: task_log_prefix_template description: | Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 6591f82dc0..86f9cf93fc 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -345,6 +345,7 @@ colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatte # Format of Log line log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s +log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware # Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter # Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}} diff --git a/airflow/utils/log/timezone_aware.py b/airflow/utils/log/timezone_aware.py new file mode 100644 index 0000000000..d01205233a --- /dev/null +++ b/airflow/utils/log/timezone_aware.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import logging + +import pendulum + + +class TimezoneAware(logging.Formatter): + """ + Override `default_time_format`, `default_msec_format` and `formatTime` to specify utc offset. + utc offset is the matter, without it, time conversion could be wrong. + With this Formatter, `%(asctime)s` will be formatted containing utc offset. (ISO 8601) + (e.g. 2022-06-12T13:00:00.123+0000) + """ + + default_time_format = '%Y-%m-%dT%H:%M:%S' + default_msec_format = '%s.%03d' + default_tz_format = '%z' + + def formatTime(self, record, datefmt=None): + """ + Returns the creation time of the specified LogRecord in ISO 8601 date and time format + in the local time zone. + """ + dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone()) + if datefmt: + s = dt.strftime(datefmt) + else: + s = dt.strftime(self.default_time_format) + + if self.default_msec_format: + s = self.default_msec_format % (s, record.msecs) + if self.default_tz_format: + s += dt.strftime(self.default_tz_format) + return s diff --git a/airflow/www/static/js/ti_log.js b/airflow/www/static/js/ti_log.js index 1bf6b501a6..ae72837345 100644 --- a/airflow/www/static/js/ti_log.js +++ b/airflow/www/static/js/ti_log.js @@ -103,6 +103,7 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { // Detect urls and log timestamps const urlRegex = /http(s)?:\/\/[\w.-]+(\.?:[\w.-]+)*([/?#][\w\-._~:/?#[\]@!$&'()*+,;=.%]+)?/g; const dateRegex = /\d{4}[./-]\d{2}[./-]\d{2} \d{2}:\d{2}:\d{2},\d{3}/g; + const iso8601Regex = /\d{4}[./-]\d{2}[./-]\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[+-]\d{4}/g; res.message.forEach((item) => { const logBlockElementId = `try-${tryNumber}-${item[0]}`; @@ -120,7 +121,8 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { const escapedMessage = escapeHtml(item[1]); const linkifiedMessage = escapedMessage .replace(urlRegex, (url) => `<a href="${url}" target="_blank">${url}</a>`) - .replaceAll(dateRegex, (date) => `<time datetime="${date}+00:00" data-with-tz="true">${formatDateTime(`${date}+00:00`)}</time>`); + .replaceAll(dateRegex, (date) => `<time datetime="${date}+00:00" data-with-tz="true">${formatDateTime(`${date}+00:00`)}</time>`) + .replaceAll(iso8601Regex, (date) => `<time datetime="${date}" data-with-tz="true">${formatDateTime(`${date}`)}</time>`); logBlock.innerHTML += `${linkifiedMessage}\n`; }); diff --git a/newsfragments/24811.significant.rst b/newsfragments/24811.significant.rst new file mode 100644 index 0000000000..cb7208843c --- /dev/null +++ b/newsfragments/24811.significant.rst @@ -0,0 +1,22 @@ +Added new config ``[logging]log_formatter_class`` to fix timezone display for logs on UI + +If you are using a custom Formatter subclass in your ``[logging]logging_config_class``, please inherit from ``airflow.utils.log.timezone_aware.TimezoneAware`` instead of ``logging.Formatter``. +For example, in your ``custom_config.py``: + +.. code-block:: python + from airflow.utils.log.timezone_aware import TimezoneAware + + # before + class YourCustomFormatter(logging.Formatter): + ... + + + # after + class YourCustomFormatter(TimezoneAware): + ... + + + AIRFLOW_FORMATTER = LOGGING_CONFIG["formatters"]["airflow"] + AIRFLOW_FORMATTER["class"] = "somewhere.your.custom_config.YourCustomFormatter" + # or use TimezoneAware class directly. If you don't have custom Formatter. + AIRFLOW_FORMATTER["class"] = "airflow.utils.log.timezone_aware.TimezoneAware"
