seanghaeli commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4631553091


##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,746 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its 
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import copy
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+    TRIGGER_ID_KEY,
+    TRIGGER_LOG_FILE_NAME,
+    TRIGGER_LOG_HELD_FILE_NAME,
+    TRIGGER_LOG_ROOT,
+    TRIGGER_LOG_ROTATED_DIR,
+    TRIGGERER_JOB_ID_KEY,
+    get_trigger_log_held_path,
+    get_trigger_log_part_path,
+    get_trigger_log_rotated_path,
+    list_trigger_log_files,
+    parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+    from structlog.typing import FilteringBoundLogger
+
+    from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory, 
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class CappedBytesLogger(structlog.BytesLogger):
+    """
+    ``BytesLogger`` appending to ``path``, moved to the rotated directory once 
it reaches ``max_bytes``.
+
+    ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; 
``max_bytes <= 0`` is no
+    cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on 
a shared log folder another
+    triggerer's orphan scan) is reopened by the next line.
+    """
+
+    __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", 
"_on_rotate", "_path", "_size")
+
+    def __init__(
+        self,
+        path: Path,
+        max_bytes: int,
+        may_rotate: Callable[[], bool] | None = None,
+        on_rotate: Callable[[], None] | None = None,
+    ):
+        self._path, self._max_bytes = path, max_bytes
+        self._may_rotate, self._on_rotate = may_rotate, on_rotate
+        self._folder_mode, self._file_mode = get_log_permissions()
+        super().__init__(path.open("ab"))
+        self._size = self._file.tell()
+
+    @property
+    def rotated_path(self) -> Path:
+        return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+    def ensure_open(self) -> None:
+        if self._file.closed:
+            self._path.parent.mkdir(mode=self._folder_mode, parents=True, 
exist_ok=True)
+            self._path.touch(mode=self._file_mode)
+            self._file = self._path.open("ab")
+            self._write = self._file.write
+            self._flush = self._file.flush
+            self._size = self._file.tell()
+
+    def _unlinked(self) -> bool:
+        try:
+            return os.fstat(self._file.fileno()).st_nlink == 0
+        except (OSError, ValueError):
+            return False
+
+    def msg(self, message: bytes) -> None:
+        with self._lock:
+            try:
+                if not self._file.closed and self._unlinked():
+                    # Lines written to a removed file are lost.
+                    with suppress(OSError):
+                        self._file.close()
+                self.ensure_open()
+                self._write(message + b"\n")
+                self._flush()
+                self._size += len(message) + 1
+                if 0 < self._max_bytes <= self._size and (self._may_rotate is 
None or self._may_rotate()):
+                    self._file.close()
+                    self.rotated_path.parent.mkdir(mode=self._folder_mode, 
parents=True, exist_ok=True)
+                    with suppress(FileNotFoundError):
+                        os.replace(self._path, self.rotated_path)
+                    if self._on_rotate is not None:
+                        try:
+                            self._on_rotate()
+                        except Exception:
+                            # The line is written: the caller, reading the 
runner's logs, must not see this.
+                            log.exception(
+                                "Failed to upload a rotated trigger log", 
path=str(self.rotated_path)
+                            )
+                    self.ensure_open()  # last: if this fails, the rotated 
lines are sent all the same
+            except OSError:
+                # Reopened by the next line: a file removed meanwhile (to free 
a full disk) frees its space.
+                with suppress(OSError):
+                    self._file.close()
+                raise
+
+    def close(self) -> None:
+        # Closing flushes; on a full disk those lines are lost either way.
+        with suppress(OSError):
+            self._file.close()
+
+    log = debug = info = warn = warning = msg
+    fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see 
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it 
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", 
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed 
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", 
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object and keep the file: 
upload_watcher_log_file empties it
+# once stored, as a file sent again would be stored twice. So here too lines 
left mean a failed write.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS
+# Remote log handlers that need none of the above. Any handler not named in 
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was 
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver 
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", 
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+    _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | 
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def get_log_permissions() -> tuple[int, int]:
+    """Return the modes of new log folders and files, as ``init_log_file`` 
gives them."""
+    return (
+        int(conf.get("logging", "file_task_handler_new_folder_permissions", 
fallback="0o775"), 8),
+        int(conf.get("logging", "file_task_handler_new_file_permissions", 
fallback="0o664"), 8),
+    )
+
+
+def _has_lines(path: Path) -> bool:
+    try:
+        return path.stat().st_size > 0
+    except OSError:
+        return False
+
+
+def handler_keeps_no_copy(handler) -> bool:
+    """
+    Whether a file the handler leaves as it was after ``upload`` is no sign of 
a failed upload.
+
+    True for a handler that streams each line as it is written (it has 
processors, like CloudWatch) or that takes
+    only task logs (Elasticsearch, OpenSearch; by class name, as nothing else 
tells them apart). Any other handler
+    that leaves the file as it was failed to upload it (S3, GCS and WASB 
report that only in their log).
+    """
+    if getattr(handler, "processors", None):
+        return True
+    return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+_handler_keeping_local_copy: tuple[object, object] | None = None
+
+
+def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None:
+    """
+    Upload one of a watcher's files, at ``path`` locally; with 
delete_local_logs, remove it once stored.
+
+    Not by the handler, which removes the whole directory of the file it 
stored: the watcher's other files are
+    in it or below it, and may not be stored yet.
+    """
+    delete = getattr(handler, "delete_local_copy", False) and not 
handler_keeps_no_copy(handler)
+    if _handler_is(handler, _APPENDING_HANDLERS):
+        # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS 
reports a failed write only
+        # through it. The file is emptied once stored (sent again, it would be 
stored twice), and kept as it
+        # was otherwise.
+        if path.is_file() and handler.oss_write(path.read_text(), 
relative_path):
+            with suppress(FileNotFoundError):
+                os.truncate(path, 0)
+    else:
+        (_get_handler_keeping_local_copy(handler) if delete else 
handler).upload(relative_path, None)
+    if delete and not _is_left_unstored(handler, path, raised=False):
+        with suppress(FileNotFoundError):
+            path.unlink()
+        with suppress(OSError):
+            path.parent.rmdir()  # once its last file is stored
+
+
+def _get_handler_keeping_local_copy(handler):
+    """Return a copy of the handler with ``delete_local_copy`` off, made once: 
it has its own hook."""
+    global _handler_keeping_local_copy
+    if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] 
is not handler:
+        keeping = copy.copy(handler)
+        keeping.delete_local_copy = False
+        _handler_keeping_local_copy = (handler, keeping)
+    return _handler_keeping_local_copy[1]
+
+
+def _append_file(lines: Path, kept: Path) -> None:
+    """Append ``lines`` to ``kept``; cut back if that fails part way (a full 
disk), as ``lines`` still has them."""
+    size = kept.stat().st_size
+    try:
+        with lines.open("rb") as source, kept.open("ab") as target:
+            shutil.copyfileobj(source, target)
+    except OSError:
+        with suppress(OSError):
+            os.truncate(kept, size)
+        raise
+
+
+def _move_into_rotated_dir(held: Path, rotated: Path) -> None:
+    rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, 
exist_ok=True)
+    os.replace(held, rotated)
+
+
+def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool:

Review Comment:
   Yes, this was deliberate. Without it, OSS could store lines twice when the 
write succeeds but emptying the local file afterwards fails, since sending the 
file again appends the same lines. Though, thanks for pointing this out, I 
looked into it, and there's a case where OSS could lose lines: when the upload 
fails before reaching OSS, for example the local file can't be read, the file 
isn't sent again and the next rotation deletes it. So now a failed upload is 
sent again for every store, and if the local file can't be emptied after OSS 
stores it, it's deleted instead.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to