o-nikolas commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4604666775


##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -564,6 +592,59 @@ def upload_to_remote(self):
         upload_to_remote(self.bound_logger, self.ti)
 
 
[email protected](kw_only=True)
+class WatcherLoggingFactory(TriggerLoggingFactory):
+    """
+    An asset watcher's own log file on this triggerer, as its owner has no log.
+
+    It is kept while the trigger is assigned here, across restarts of the 
watcher (SQS-style ones restart
+    after every event), so it is appended to, and capped on local disk. It is 
uploaded with the remote log
+    handler task logs use, and every line keeps its ``trigger_id``.
+    """
+
+    max_bytes: int = 0
+    to_stdout: bool = True
+    # Told about every line, to cap how many of these files are open at once.
+    on_write: Callable[[int], None] | None = attrs.field(default=None, 
repr=False)
+    may_rotate: Callable[[], bool] | None = attrs.field(default=None, 
repr=False)
+    on_rotate: Callable[[], None] | None = attrs.field(default=None, 
repr=False)
+
+    def _open(self, processors: Iterable[structlog.typing.Processor]) -> None:
+        if hasattr(self, "bound_logger"):
+            # Before the processors run: a handler streaming each line 
(CloudWatch) names its stream after the
+            # open file, and would drop the line of a file the open-files cap 
closed.
+            self.bound_logger._logger.ensure_open()
+        else:
+            from airflow.sdk.log import init_log_file  # noqa: SDK001
+
+            underlying_logger = _CappedBytesLogger(
+                init_log_file(self.log_path), self.max_bytes, self.may_rotate, 
self.on_rotate
+            )
+            self.bound_logger = structlog.wrap_logger(underlying_logger, 
processors=processors).bind(
+                trigger_id=self.trigger_id
+            )
+        if self.on_write is not None and self.trigger_id is not None:
+            self.on_write(self.trigger_id)
+
+    def close(self):
+        if hasattr(self, "bound_logger"):
+            self.bound_logger._logger.close()
+
+    def upload_to_remote(self):
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        if not hasattr(self, "bound_logger") or (handler := 
load_remote_log_handler()) is None:
+            return
+        # Never upload an open file: remote handlers empty or delete it 
afterwards. The rotated files first, as
+        # remote handlers append in upload order. Nor an empty one to a 
handler that stores files: it would
+        # download and upload the remote copy again for nothing.
+        self.close()
+        live = self.bound_logger._logger._path
+        for path in list_trigger_log_files(live.parent):
+            if _handler_keeps_no_copy(handler) or (path.is_file() and 
path.stat().st_size):

Review Comment:
   +1. One more path to the same duplicate: a watcher that comes back to the 
same triggerer after its release upload finished gets (True, None) from 
take_back, so the new factory reopens the same trigger.log in append mode with 
the old lines still in it, and the next release sends them to OSS again. Since 
delete_local_logs is off by default, this hits OSS in its default setup.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to