(airflow) branch main updated: [Task] add NonCachingRotatingFileHanlder for worker task (#41064)

[potiuk]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new b6ed1a9be4 [Task] add NonCachingRotatingFileHanlder for worker task 
(#41064)
b6ed1a9be4 is described below

commit b6ed1a9be41bf76c6eb919f40523fb78f2021f61
Author: Huanjie Guo <huanjie....@outlook.com>
AuthorDate: Sun Jul 28 18:40:33 2024 +0800

    [Task] add NonCachingRotatingFileHanlder for worker task (#41064)
    
    * [Task] add NonCachingRotatingFileHanlder for worker task
    
    * [Task] format the change
---
 airflow/utils/log/file_task_handler.py             | 25 +++++++++++++++++++---
 .../advanced-logging-configuration.rst             | 21 ++++++++++++++++++
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/airflow/utils/log/file_task_handler.py 
b/airflow/utils/log/file_task_handler.py
index e177d232d5..fa43d7f8d6 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -39,7 +39,7 @@ from airflow.executors.executor_loader import ExecutorLoader
 from airflow.utils.context import Context
 from airflow.utils.helpers import parse_template_string, 
render_template_to_string
 from airflow.utils.log.logging_mixin import SetContextPropagate
-from airflow.utils.log.non_caching_file_handler import NonCachingFileHandler
+from airflow.utils.log.non_caching_file_handler import 
NonCachingRotatingFileHandler
 from airflow.utils.session import provide_session
 from airflow.utils.state import State, TaskInstanceState
 
@@ -176,6 +176,9 @@ class FileTaskHandler(logging.Handler):
 
     :param base_log_folder: Base log folder to place logs.
     :param filename_template: template filename string
+    :param max_bytes: max bytes size for the log file
+    :param backup_count: backup file count for the log file
+    :param delay:  default False -> StreamHandler, True -> Handler
     """
 
     trigger_should_wrap = True
@@ -183,7 +186,14 @@ class FileTaskHandler(logging.Handler):
         "Operator inherits from empty operator and thus does not have logs"
     )
 
-    def __init__(self, base_log_folder: str, filename_template: str | None = 
None):
+    def __init__(
+        self,
+        base_log_folder: str,
+        filename_template: str | None = None,
+        max_bytes: int = 0,
+        backup_count: int = 0,
+        delay: bool = False,
+    ):
         super().__init__()
         self.handler: logging.Handler | None = None
         self.local_base = base_log_folder
@@ -196,6 +206,9 @@ class FileTaskHandler(logging.Handler):
                 stacklevel=(2 if type(self) == FileTaskHandler else 3),
             )
         self.maintain_propagate: bool = False
+        self.max_bytes = max_bytes
+        self.backup_count = backup_count
+        self.delay = delay
         """
         If true, overrides default behavior of setting propagate=False
 
@@ -224,7 +237,13 @@ class FileTaskHandler(logging.Handler):
             to task logs from a context other than task or trigger run
         """
         local_loc = self._init_file(ti, identifier=identifier)
-        self.handler = NonCachingFileHandler(local_loc, encoding="utf-8")
+        self.handler = NonCachingRotatingFileHandler(
+            local_loc,
+            encoding="utf-8",
+            maxBytes=self.max_bytes,
+            backupCount=self.backup_count,
+            delay=self.delay,
+        )
         if self.formatter:
             self.handler.setFormatter(self.formatter)
         self.handler.setLevel(self.level)
diff --git 
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst
 
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst
index 0bffb59402..6339735664 100644
--- 
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst
+++ 
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst
@@ -161,3 +161,24 @@ Example of custom logger name:
               }
           },
       )
+
+If you want to limit the log size of the tasks, you can add the 
handlers.task.max_bytes parameter.
+
+Example of limiting the size of tasks:
+
+    .. code-block:: python
+
+      from copy import deepcopy
+      from pydantic.utils import deep_update
+      from airflow.config_templates.airflow_local_settings import 
DEFAULT_LOGGING_CONFIG
+
+      LOGGING_CONFIG = deep_update(
+          deepcopy(DEFAULT_LOGGING_CONFIG),
+          {
+              "handlers": {
+                  "task": {
+                      "max_bytes": 104857600,  # 100MB
+                  }
+              }
+          },
+      )