seanghaeli commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4625194620


##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -564,6 +601,76 @@ def upload_to_remote(self):
         upload_to_remote(self.bound_logger, self.ti)
 
 
[email protected](kw_only=True)
+class WatcherLoggingFactory(TriggerLoggingFactory):
+    """
+    An asset watcher's own log file on this triggerer, as its owner has no log.
+
+    It is kept while the trigger is assigned here, across restarts of the 
watcher (SQS-style ones restart
+    after every event), so it is appended to, and capped on local disk. It is 
uploaded with the remote log
+    handler task logs use, and every line keeps its ``trigger_id``.
+    """
+
+    max_bytes: int = 0
+    to_stdout: bool = True
+    # Told about every line, to cap how many of these files are open at once.
+    on_write: Callable[[int], None] | None = attrs.field(default=None, 
repr=False)
+    may_rotate: Callable[[], bool] | None = attrs.field(default=None, 
repr=False)
+    on_rotate: Callable[[], None] | None = attrs.field(default=None, 
repr=False)
+
+    def _open(self, processors: Iterable[structlog.typing.Processor]) -> None:
+        if hasattr(self, "bound_logger"):
+            # Before the processors run: a handler streaming each line 
(CloudWatch) names its stream after the
+            # open file, and would drop the line of a file the open-files cap 
closed.
+            self.bound_logger._logger.ensure_open()
+        else:
+            from airflow.sdk.log import init_log_file  # noqa: SDK001
+
+            underlying_logger = CappedBytesLogger(
+                init_log_file(self.log_path), self.max_bytes, self.may_rotate, 
self.on_rotate
+            )
+            self.bound_logger = structlog.wrap_logger(underlying_logger, 
processors=processors).bind(
+                trigger_id=self.trigger_id
+            )
+        if self.on_write is not None and self.trigger_id is not None:
+            self.on_write(self.trigger_id)
+
+    def close(self):
+        if hasattr(self, "bound_logger"):
+            self.bound_logger._logger.close()
+
+    def upload_to_remote(
+        self, *, keep_live: bool = False, take_held: Callable[[], Path | None] 
| None = None
+    ):
+        """
+        Upload the watcher's files on this triggerer.
+
+        ``take_held`` moves the held file (see ``WatcherLogs.release``) into 
the rotated directory and returns
+        where; without it, the held file is not sent. With ``keep_live``, the 
live file is not sent either: a
+        watcher back here writes it.
+        """
+        from airflow.sdk.log import load_remote_log_handler  # noqa: SDK001
+
+        if not hasattr(self, "bound_logger") or (handler := 
load_remote_log_handler()) is None:
+            return
+        # Never upload an open file: remote handlers empty or delete it 
afterwards. The rotated files first, as
+        # remote handlers append in upload order. Nor an empty one to a 
handler that stores files: it would
+        # download and upload the remote copy again for nothing.
+        self.close()
+        live = self.bound_logger._logger._path
+        stores_files = not handler_keeps_no_copy(handler)
+        for local in list_trigger_log_files(live.parent):
+            path: Path | None = local
+            if local.name == TRIGGER_LOG_HELD_FILE_NAME:
+                # Only now: with delete_local_logs, the uploads of the rotated 
files before it remove their directory.
+                path = take_held() if take_held is not None else None
+            if path is None or (keep_live and local == live):
+                continue
+            relative_path = (Path(self.log_path).parent / 
path.relative_to(live.parent)).as_posix()
+            if not stores_files or (path.is_file() and path.stat().st_size):
+                upload_watcher_log_file(handler, relative_path, path)

Review Comment:
   This could happen on any upload, not only the live file's: each one could 
delete a neighbour that hadn't been stored yet. Now a file is removed only once 
it's stored, so a part whose upload failed stays for the next upload whatever 
goes through after it, at a stop and in the orphan scan.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to