seanghaeli commented on code in PR #76527: URL: https://github.com/apache/airflow/pull/76527#discussion_r4641346389
########## airflow-core/src/airflow/utils/log/watcher_log.py: ########## @@ -0,0 +1,746 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""An asset watcher's own log file on the triggerer: the capped file, its uploads and the orphan scan.""" + +from __future__ import annotations + +import copy +import os +import shutil +import threading +import time +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING + +import attrs +import structlog +from sqlalchemy import select + +from airflow.configuration import conf +from airflow.models.trigger import Trigger +from airflow.utils.helpers import chunks +from airflow.utils.log.trigger_log import ( + TRIGGER_ID_KEY, + TRIGGER_LOG_FILE_NAME, + TRIGGER_LOG_HELD_FILE_NAME, + TRIGGER_LOG_ROOT, + TRIGGER_LOG_ROTATED_DIR, + TRIGGERER_JOB_ID_KEY, + get_trigger_log_held_path, + get_trigger_log_part_path, + get_trigger_log_rotated_path, + list_trigger_log_files, + parse_trigger_log_dir_id, +) +from airflow.utils.session import create_session + +if TYPE_CHECKING: + from structlog.typing import FilteringBoundLogger + + from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory + +log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__) + + +class CappedBytesLogger(structlog.BytesLogger): + """ + ``BytesLogger`` appending to ``path``, moved to the rotated directory once it reaches ``max_bytes``. + + ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; ``max_bytes <= 0`` is no + cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on a shared log folder another + triggerer's orphan scan) is reopened by the next line. + """ + + __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", "_on_rotate", "_path", "_size") + + def __init__( + self, + path: Path, + max_bytes: int, + may_rotate: Callable[[], bool] | None = None, + on_rotate: Callable[[], None] | None = None, + ): + self._path, self._max_bytes = path, max_bytes + self._may_rotate, self._on_rotate = may_rotate, on_rotate + self._folder_mode, self._file_mode = get_log_permissions() + super().__init__(path.open("ab")) + self._size = self._file.tell() + + @property + def rotated_path(self) -> Path: + return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name + + def ensure_open(self) -> None: + if self._file.closed: + self._path.parent.mkdir(mode=self._folder_mode, parents=True, exist_ok=True) + self._path.touch(mode=self._file_mode) + self._file = self._path.open("ab") + self._write = self._file.write + self._flush = self._file.flush + self._size = self._file.tell() + + def _unlinked(self) -> bool: + try: + return os.fstat(self._file.fileno()).st_nlink == 0 + except (OSError, ValueError): + return False + + def msg(self, message: bytes) -> None: + with self._lock: + try: + if not self._file.closed and self._unlinked(): + # Lines written to a removed file are lost. + with suppress(OSError): + self._file.close() + self.ensure_open() + self._write(message + b"\n") + self._flush() + self._size += len(message) + 1 + if 0 < self._max_bytes <= self._size and (self._may_rotate is None or self._may_rotate()): + self._file.close() + self.rotated_path.parent.mkdir(mode=self._folder_mode, parents=True, exist_ok=True) + with suppress(FileNotFoundError): + os.replace(self._path, self.rotated_path) + if self._on_rotate is not None: + try: + self._on_rotate() + except Exception: + # The line is written: the caller, reading the runner's logs, must not see this. + log.exception( + "Failed to upload a rotated trigger log", path=str(self.rotated_path) + ) + self.ensure_open() # last: if this fails, the rotated lines are sent all the same + except OSError: + # Reopened by the next line: a file removed meanwhile (to free a full disk) frees its space. + with suppress(OSError): + self._file.close() + raise + + def close(self) -> None: + # Closing flushes; on a full disk those lines are lost either way. + with suppress(OSError): + self._file.close() + + log = debug = info = warn = warning = msg + fatal = failure = err = error = critical = exception = msg + + +# Marks a dead triggerer's log directory that one triggerer is uploading (see handle_orphaned_trigger_logs). +_ORPHAN_UPLOAD_CLAIM = ".uploading" +_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0 +_TRIGGER_ID_QUERY_CHUNK_SIZE = 500 +# Remote log handlers that take only task logs: a watcher's file is left as it was however often it is sent. +_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", "OpensearchRemoteLogIO"}) +# Remote log handlers that empty a file they stored and only log a failed write: lines left mean it failed. +_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", "WasbRemoteLogIO"}) +# Remote log handlers that append to the stored object and keep the file: upload_watcher_log_file empties it +# once stored, as a file sent again would be stored twice. So here too lines left mean a failed write. +_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"}) +_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | _APPENDING_HANDLERS +# Remote log handlers that need none of the above. Any handler not named in these lists is logged once (see +# _handler_is): a renamed one would otherwise silently lose what its name was listed for. +# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver is a Cloud Logging subclass). +_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", "CloudLoggingRemoteLogIO"}) +_KNOWN_HANDLERS = ( + _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | _APPENDING_HANDLERS | _PLAIN_HANDLERS +) +_checked_handler_classes: set[type] = set() + + +def get_log_permissions() -> tuple[int, int]: + """Return the modes of new log folders and files, as ``init_log_file`` gives them.""" + return ( + int(conf.get("logging", "file_task_handler_new_folder_permissions", fallback="0o775"), 8), + int(conf.get("logging", "file_task_handler_new_file_permissions", fallback="0o664"), 8), + ) + + +def _has_lines(path: Path) -> bool: + try: + return path.stat().st_size > 0 + except OSError: + return False + + +def handler_keeps_no_copy(handler) -> bool: + """ + Whether a file the handler leaves as it was after ``upload`` is no sign of a failed upload. + + True for a handler that streams each line as it is written (it has processors, like CloudWatch) or that takes + only task logs (Elasticsearch, OpenSearch; by class name, as nothing else tells them apart). Any other handler + that leaves the file as it was failed to upload it (S3, GCS and WASB report that only in their log). + """ + if getattr(handler, "processors", None): + return True + return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS) + + +_handler_keeping_local_copy: tuple[object, object] | None = None + + +def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None: + """ + Upload one of a watcher's files, at ``path`` locally; with delete_local_logs, remove it once stored. + + Not by the handler, which removes the whole directory of the file it stored: the watcher's other files are + in it or below it, and may not be stored yet. + """ + delete = getattr(handler, "delete_local_copy", False) and not handler_keeps_no_copy(handler) + if _handler_is(handler, _APPENDING_HANDLERS): + # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a failed write only + # through it. The file is emptied once stored, or removed if it can't be (sent again, it would be + # stored twice), and kept as it was otherwise. + if path.is_file() and handler.oss_write(path.read_text(), relative_path): + try: + os.truncate(path, 0) + except OSError: + path.unlink(missing_ok=True) + else: + (_get_handler_keeping_local_copy(handler) if delete else handler).upload(relative_path, None) + if delete and not _is_left_unstored(handler, path, raised=False): + with suppress(FileNotFoundError): + path.unlink() + with suppress(OSError): + path.parent.rmdir() # once its last file is stored + + +def _get_handler_keeping_local_copy(handler): + """Return a copy of the handler with ``delete_local_copy`` off, made once: it has its own hook.""" + global _handler_keeping_local_copy + if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] is not handler: + keeping = copy.copy(handler) + keeping.delete_local_copy = False + _handler_keeping_local_copy = (handler, keeping) + return _handler_keeping_local_copy[1] + + +def _append_file(lines: Path, kept: Path) -> None: + """Append ``lines`` to ``kept``; cut back if that fails part way (a full disk), as ``lines`` still has them.""" + size = kept.stat().st_size + try: + with lines.open("rb") as source, kept.open("ab") as target: + shutil.copyfileobj(source, target) + except OSError: + with suppress(OSError): + os.truncate(kept, size) + raise + + +def _move_into_rotated_dir(held: Path, rotated: Path) -> None: + rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, exist_ok=True) + os.replace(held, rotated) + + +def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool: + """ + Whether to send a watcher's file again after an upload. + + True if it has lines and the upload raised, or S3, GCS, WASB or OSS left them (others may keep them). + """ + if not _has_lines(path): + return False + return raised or _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS) + + +def _handler_is(handler, names: frozenset[str]) -> bool: + """Whether the handler's class, or a base class, has one of these names (as for the above).""" + classes = type(handler).__mro__ + if type(handler) not in _checked_handler_classes: + _checked_handler_classes.add(type(handler)) + if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes): + log.info( + "Remote log handler not known by name: asset watcher log files are sent to it as to one " + "that stores them, and sent again only if the upload raises", + handler=f"{type(handler).__module__}.{type(handler).__qualname__}", + ) + return any(cls.__name__ in names for cls in classes) + + +def _get_existing_trigger_ids(session, trigger_ids: set[int]) -> set[int]: + return { + trigger_id + for ids in chunks(sorted(trigger_ids), _TRIGGER_ID_QUERY_CHUNK_SIZE) + for trigger_id in session.scalars(select(Trigger.id).where(Trigger.id.in_(ids))) + } + + +def get_watcher_log_max_open_files() -> int: + """ + Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a quarter of ``ulimit -n``. + + The task and callback trigger logs, the database and the sockets need descriptors too. + """ + setting = conf.getint("triggerer", "watcher_log_max_open_files") + try: + import resource # Unix only + + soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (ImportError, OSError, ValueError): + return setting + if soft == resource.RLIM_INFINITY: + return setting + limit = max(1, soft // 4) + return limit if setting <= 0 else min(setting, limit) + + [email protected](kw_only=True, eq=False) +class _Release: + """The upload of a released watcher's files.""" + + factory: TriggerLoggingFactory + # The upload of the same watcher's files released before, if it was still pending + previous: _Release | None + upload: Future = attrs.field(init=False) + + [email protected](kw_only=True) +class WatcherLogs: + """ + The asset watcher log files of one triggerer's supervisor, other than writing to them. + + Keeps at most ``watcher_log_max_open_files`` of them open, uploads their rotated files and those of + released watchers off the main loop, and uploads or removes the files triggerers that died left here. + """ + + # The supervisor's trigger log factories, by trigger id + logger_cache: dict[int, TriggerLoggingFactory] + # Told when closing a trigger's log file fails + on_error: Callable[[int | None, OSError], None] + # Asset watcher files with an open descriptor, least recently written first + _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False) + watcher_log_max_open_files: int = attrs.field(factory=get_watcher_log_max_open_files, init=False) + # When to look for watcher files left by triggerers that died, and the thread doing it + _orphan_scans_at: list[float] | None = attrs.field(default=None, init=False) + _orphan_scan_thread: threading.Thread | None = attrs.field(default=None, init=False) + # Uploads of released watchers' files, and of rotated files in their own pool (release uploads wait) + _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None, init=False) + _releases: dict[int, _Release] = attrs.field(factory=dict, init=False) + # Held to store a release, and to forget it on its upload's thread + _releases_lock: threading.Lock = attrs.field(factory=threading.Lock, init=False) + # Log paths of watchers that left while their trigger existed, removed here once it no longer does + _left: dict[int, str] = attrs.field(factory=dict, init=False) + _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None, init=False) + _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict, init=False) + # The last number given to each watcher's rotated files here (see upload_rotated) + _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False) + # Each watcher's last rotated file whose upload the handler did not confirm, sent again before it is replaced + _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict, init=False) + # Held while a released watcher's file is added to its held file, or the held file is taken for upload + _held_lock: threading.Lock = attrs.field(factory=threading.Lock, init=False) + + def note_watcher_log_write(self, trigger_id: int) -> None: + """Keep at most ``watcher_log_max_open_files`` watcher files open, closing the least recent.""" + self._open_watcher_logs.pop(trigger_id, None) + self._open_watcher_logs[trigger_id] = None + while 0 < self.watcher_log_max_open_files < len(self._open_watcher_logs): + idle = next(iter(self._open_watcher_logs)) + del self._open_watcher_logs[idle] + if (factory := self.logger_cache.get(idle)) is not None: + factory.close() + + def forget_open_file(self, trigger_id: int) -> None: + """Stop counting a trigger's file as open, as it is about to be closed.""" + self._open_watcher_logs.pop(trigger_id, None) + + def release(self, trigger_id: int, factory: TriggerLoggingFactory) -> None: + """ + Upload and close the file of a watcher no longer assigned here, off the main loop. + + The watcher may come back here at once, on a new live file: this one is held next to it until an upload + of the watcher's files here takes it (see _hold). + """ + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + # In the background: a triggerer that lost its watchers (a DB partition) releases them all at once. + if self._upload_pool is None: + self._upload_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="trigger-log-release") + handler = load_remote_log_handler() + if handler is not None and not _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS): + if hasattr(factory, "bound_logger"): + factory.close() + self._hold(factory.bound_logger._logger._path) + with self._releases_lock: + release = _Release(factory=factory, previous=self._releases.get(trigger_id)) + release.upload = self._upload_pool.submit(self._upload_released, trigger_id, release) + self._releases[trigger_id] = release + + def forget(done: Future) -> None: + with self._releases_lock: + if self._releases.get(trigger_id) is release: + self._releases.pop(trigger_id, None) + + release.upload.add_done_callback(forget) + + def _hold(self, live: Path) -> None: + """Move a released watcher's lines out of the way of a watcher back here (see release).""" + held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME) + # Else left in the live file: a watcher back here appends to it, and its next upload sends them. + with self._held_lock, suppress(OSError): + if not _has_lines(live): + return + if not held.exists(): + os.replace(live, held) + return + # Released again before the held file was taken for upload: one file, sent once. + _append_file(live, held) + live.unlink() + + def _take_held(self, trigger_id: int, log_path: str) -> str | None: + """Move the held file into the rotated directory for upload; return its path there, if there was one.""" + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + base = Path(conf.get("logging", "base_log_folder")) + held = base / get_trigger_log_held_path(log_path) + if (handler := load_remote_log_handler()) is None or not held.is_file(): + return None + if handler_keeps_no_copy(handler): + path = get_trigger_log_rotated_path(log_path) + else: + # Numbered only now, as the watcher's rotations wait for this upload (see prepare_rotation). + part = self._rotated_parts[trigger_id] = self._rotated_parts.get(trigger_id, 0) + 1 + path = get_trigger_log_part_path(log_path, part) + with self._held_lock: + _move_into_rotated_dir(held, base / path) + return path + + def _upload_released(self, trigger_id: int, release: _Release) -> None: + if release.previous is not None: + # Its files are in the same directory. + with suppress(Exception): + release.previous.upload.result() + release.previous = None + factory = release.factory + failed = self._upload_and_close(trigger_id, factory, keep_live=True) + if failed: + self._left.pop(trigger_id, None) # kept for the next start's orphan scan + else: + self._left[trigger_id] = factory.log_path + + @staticmethod + def _remove_released_files(log_path: str) -> None: + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + producer_dir = (Path(conf.get("logging", "base_log_folder")) / log_path).parent + handler = load_remote_log_handler() + if handler is not None and _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS): + if any(_has_lines(path) for path in list_trigger_log_files(producer_dir)): + return # a failed upload: sent again by the next start's orphan scan + shutil.rmtree(producer_dir, ignore_errors=True) + with suppress(OSError): + producer_dir.parent.rmdir() # not while another triggerer still has a directory in it + + def remove_left(self) -> None: + """Remove the files of watchers that left here and whose trigger no longer exists.""" + if not (left := set(self._left)): + return + try: + with create_session() as session: + gone = left - _get_existing_trigger_ids(session, left) + except Exception: + log.exception("Failed to look for triggers that no longer exist") + return + for trigger_id in gone: + if (release := self._releases.get(trigger_id)) is not None and not release.upload.done(): + continue # its files are still uploading + log_path = self._left.pop(trigger_id, None) + if log_path is None or trigger_id in self.logger_cache: + continue # back here: tracked again once it leaves + try: + self._remove_released_files(log_path) + except Exception: + log.exception("Failed to remove the log files of a trigger that no longer exists") + + def prepare_rotation(self, trigger_id: int) -> bool: + """Return whether the watcher's file may rotate now; until then, a part left unsent is sent again.""" + if (release := self._releases.get(trigger_id)) is not None and not release.upload.done(): + return False # its released files are uploading from the rotated files' directory + upload = self._pending_rotated_uploads.get(trigger_id) + if upload is not None and not upload.done(): + return False + if path := self._rotated_unconfirmed.pop(trigger_id, None): + # Before the next rotation, which replaces it whether this goes through or not. + self._pending_rotated_uploads[trigger_id] = self._get_rotated_pool().submit( + self._upload_rotated_again, trigger_id, path + ) + return False + return True + + def _get_rotated_pool(self) -> ThreadPoolExecutor: + if self._rotated_pool is None: + self._rotated_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="trigger-log-rotated") + return self._rotated_pool + + @staticmethod + def _upload_rotated_again(trigger_id: int, path: str) -> None: + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + try: + if (handler := load_remote_log_handler()) is not None: + upload_watcher_log_file(handler, path, Path(conf.get("logging", "base_log_folder")) / path) + except Exception: + log.exception("Failed to upload a rotated trigger log", trigger_id=trigger_id, path=path) + + def upload_rotated(self, trigger_id: int, log_path: str) -> None: + # Called by the line that rotated the file: the rest runs in the pool, not to hold up the runner's logs. + pool = self._get_rotated_pool() + rotated = get_trigger_log_rotated_path(log_path) + # Numbered here: a watcher's next rotation waits for this upload (see prepare_rotation). + part = self._rotated_parts.get(trigger_id, 0) + 1 + + def upload() -> None: + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + path = rotated + base = Path(conf.get("logging", "base_log_folder")) + try: + if (handler := load_remote_log_handler()) is None: + return + if not handler_keeps_no_copy(handler): + # Each rotated file is its own object (see get_trigger_log_part_path), numbered by this + # triggerer, the only one writing there: counted, as earlier parts may be gone locally. + for done in (base / rotated).parent.glob("*.log"): + if done.name != TRIGGER_LOG_FILE_NAME: + with suppress(OSError): + done.unlink() # the previous one, uploaded or tried twice: one at a time + path = get_trigger_log_part_path(log_path, part) + # Named only now: until then the triggerer's log server serves it as the rotated file. + os.replace(base / rotated, base / path) + upload_watcher_log_file(handler, path, base / path) + raised = False + except Exception: + log.exception("Failed to upload a rotated trigger log", trigger_id=trigger_id, path=path) + raised = True + if path != rotated and _is_left_unstored(handler, base / path, raised=raised): + self._rotated_unconfirmed[trigger_id] = path + + self._pending_rotated_uploads[trigger_id] = pool.submit(upload) + self._rotated_parts[trigger_id] = part + + def upload_and_close(self, trigger_id: int, factory: TriggerLoggingFactory) -> None: + if (release := self._releases.get(trigger_id)) is not None: + # A watcher back here: its released files are uploading from the same directory. + with suppress(Exception): + release.upload.result() + self._upload_and_close(trigger_id, factory) + + def _upload_and_close(self, trigger_id: int, factory: TriggerLoggingFactory, **upload_kwargs) -> bool: + """Upload and close a trigger's file; return whether the upload raised.""" + from airflow.jobs.triggerer_job_runner import WatcherLoggingFactory + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id, None)) is not None: + rotated_upload.result() # else the rotated file could be read again before it is emptied + self._rotated_unconfirmed.pop(trigger_id, None) # sent again below, with the live file + base = Path(conf.get("logging", "base_log_folder")) + failed, held = False, None + try: + # Not by a factory that never wrote: its upload sends nothing, so the held file waits. + if isinstance(factory, WatcherLoggingFactory) and hasattr(factory, "bound_logger"): + held = self._take_held(trigger_id, factory.log_path) + factory.upload_to_remote(**upload_kwargs) + except Exception: + log.exception("Failed to upload trigger logs to remote", trigger_id=trigger_id) + failed = True + finally: + # Close even if the upload failed, not to leak the descriptor. Closing flushes, which can fail. + try: + factory.close() + except OSError as e: + self.on_error(trigger_id, e) + if held and _is_left_unstored(load_remote_log_handler(), base / held, raised=failed): + # Under a rotated file's name now: the watcher's next rotation here would remove it unsent. + self._rotated_unconfirmed[trigger_id] = held + return failed + + def upload_and_close_all(self, factories: list[tuple[int, TriggerLoggingFactory]]) -> None: + """Upload and close these watchers' files, and wait for the released ones' uploads.""" + self._open_watcher_logs.clear() + # Side by side: one after the other, thousands of files take longer than a container's stop timeout. + with ThreadPoolExecutor(max_workers=16, thread_name_prefix="trigger-log-upload") as pool: + list(pool.map(lambda item: self.upload_and_close(*item), factories)) + if self._upload_pool is not None: + self._upload_pool.shutdown(wait=True) + + def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None: + """ + Upload, or remove, the asset watcher files left on this disk by triggerers that are no longer alive. + + A killed triggerer uploads nothing, so its files are uploaded now, then their directories removed. + Files remote logging doesn't take (it is off, or the handler keeps no copy, like Elasticsearch) are + removed instead: all but the most recently written for each watcher, and all for triggers that no + longer exist. Files of live triggerers, this one included, of triggerers without a job (no job id + directory), and those a failed upload left stay. + """ + from airflow.jobs.job import Job + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + handler = load_remote_log_handler() + base = Path(conf.get("logging", "base_log_folder")) + dirs_by_job: dict[int, list[Path]] = {} + dirs_by_trigger: dict[Path, int] = {} + for producer_dir in base.glob(f"{TRIGGER_LOG_ROOT}/{TRIGGER_ID_KEY}=*/{TRIGGERER_JOB_ID_KEY}=*"): + # Only directories this module names are ever removed: both ids must parse. + trigger_id = parse_trigger_log_dir_id(TRIGGER_ID_KEY, producer_dir.parent.name) + job_id = parse_trigger_log_dir_id(TRIGGERER_JOB_ID_KEY, producer_dir.name) + if ( + trigger_id is not None + and job_id is not None + and job_id != own_job_id + and producer_dir.is_dir() + ): + dirs_by_job.setdefault(job_id, []).append(producer_dir) + dirs_by_trigger[producer_dir.parent] = trigger_id + if not dirs_by_job: + return + with create_session() as session: + alive = session.scalars( + select(Job.id).where(Job.id.in_(dirs_by_job), Job.is_alive_clause(["TriggererJob"])) + ) + dead = dirs_by_job.keys() - set(alive) + # A watcher whose kwargs changed got a new trigger; the old id's files have nothing left to keep. + existing = _get_existing_trigger_ids(session, set(dirs_by_trigger.values())) + dead_dirs = {job_id: dirs_by_job[job_id] for job_id in dead} + if handler is not None: + kept = { + producer_dir: self._upload_orphaned_trigger_log_dir(handler, base, producer_dir, job_id) + for job_id, dirs in dead_dirs.items() + for producer_dir in dirs + } + # What the handler left as it was is then removed as without remote logging. + dead_dirs = {job_id: [d for d in dirs if kept[d]] for job_id, dirs in dead_dirs.items()} + if None in kept.values() and self._orphan_scans_at is not None: + # Claimed by a triggerer that may have stopped mid-scan (a container restarted in place, so no + # later start may come): looked at again once the claim is stale. + self._orphan_scans_at.append(time.monotonic() + _ORPHAN_UPLOAD_CLAIM_MAX_AGE) + gone = { + trigger_dir for trigger_dir, trigger_id in dirs_by_trigger.items() if trigger_id not in existing + } + self._remove_old_trigger_log_dirs(dead_dirs, gone=gone) + + @staticmethod + def _upload_orphaned_trigger_log_dir(handler, base: Path, producer_dir: Path, job_id: int) -> bool | None: + """Upload a dead triggerer's files: True if the handler left any, None if they are claimed.""" + # Claimed first: triggerers starting together on a shared log folder see the same files, and remote + # handlers append, so a file uploaded twice would be in the remote copy twice. + claim = producer_dir / _ORPHAN_UPLOAD_CLAIM + with suppress(OSError): + if time.time() - claim.stat().st_mtime >= _ORPHAN_UPLOAD_CLAIM_MAX_AGE: Review Comment: Fixed: a stale claim is now moved aside by rename before a scan takes it over, so only one scan can win. Added a test where a second scan takes it over while the first is uploading. ########## airflow-core/src/airflow/utils/log/watcher_log.py: ########## @@ -0,0 +1,746 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""An asset watcher's own log file on the triggerer: the capped file, its uploads and the orphan scan.""" + +from __future__ import annotations + +import copy +import os +import shutil +import threading +import time +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING + +import attrs +import structlog +from sqlalchemy import select + +from airflow.configuration import conf +from airflow.models.trigger import Trigger +from airflow.utils.helpers import chunks +from airflow.utils.log.trigger_log import ( + TRIGGER_ID_KEY, + TRIGGER_LOG_FILE_NAME, + TRIGGER_LOG_HELD_FILE_NAME, + TRIGGER_LOG_ROOT, + TRIGGER_LOG_ROTATED_DIR, + TRIGGERER_JOB_ID_KEY, + get_trigger_log_held_path, + get_trigger_log_part_path, + get_trigger_log_rotated_path, + list_trigger_log_files, + parse_trigger_log_dir_id, +) +from airflow.utils.session import create_session + +if TYPE_CHECKING: + from structlog.typing import FilteringBoundLogger + + from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory + +log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__) + + +class CappedBytesLogger(structlog.BytesLogger): + """ + ``BytesLogger`` appending to ``path``, moved to the rotated directory once it reaches ``max_bytes``. + + ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one; ``max_bytes <= 0`` is no + cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on a shared log folder another + triggerer's orphan scan) is reopened by the next line. + """ + + __slots__ = ("_file_mode", "_folder_mode", "_max_bytes", "_may_rotate", "_on_rotate", "_path", "_size") + + def __init__( + self, + path: Path, + max_bytes: int, + may_rotate: Callable[[], bool] | None = None, + on_rotate: Callable[[], None] | None = None, + ): + self._path, self._max_bytes = path, max_bytes + self._may_rotate, self._on_rotate = may_rotate, on_rotate + self._folder_mode, self._file_mode = get_log_permissions() + super().__init__(path.open("ab")) + self._size = self._file.tell() + + @property + def rotated_path(self) -> Path: + return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name + + def ensure_open(self) -> None: + if self._file.closed: + self._path.parent.mkdir(mode=self._folder_mode, parents=True, exist_ok=True) + self._path.touch(mode=self._file_mode) + self._file = self._path.open("ab") + self._write = self._file.write + self._flush = self._file.flush + self._size = self._file.tell() + + def _unlinked(self) -> bool: + try: + return os.fstat(self._file.fileno()).st_nlink == 0 + except (OSError, ValueError): + return False + + def msg(self, message: bytes) -> None: + with self._lock: + try: + if not self._file.closed and self._unlinked(): + # Lines written to a removed file are lost. + with suppress(OSError): + self._file.close() + self.ensure_open() + self._write(message + b"\n") + self._flush() + self._size += len(message) + 1 + if 0 < self._max_bytes <= self._size and (self._may_rotate is None or self._may_rotate()): + self._file.close() + self.rotated_path.parent.mkdir(mode=self._folder_mode, parents=True, exist_ok=True) + with suppress(FileNotFoundError): + os.replace(self._path, self.rotated_path) + if self._on_rotate is not None: + try: + self._on_rotate() + except Exception: + # The line is written: the caller, reading the runner's logs, must not see this. + log.exception( + "Failed to upload a rotated trigger log", path=str(self.rotated_path) + ) + self.ensure_open() # last: if this fails, the rotated lines are sent all the same + except OSError: + # Reopened by the next line: a file removed meanwhile (to free a full disk) frees its space. + with suppress(OSError): + self._file.close() + raise + + def close(self) -> None: + # Closing flushes; on a full disk those lines are lost either way. + with suppress(OSError): + self._file.close() + + log = debug = info = warn = warning = msg + fatal = failure = err = error = critical = exception = msg + + +# Marks a dead triggerer's log directory that one triggerer is uploading (see handle_orphaned_trigger_logs). +_ORPHAN_UPLOAD_CLAIM = ".uploading" +_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0 +_TRIGGER_ID_QUERY_CHUNK_SIZE = 500 +# Remote log handlers that take only task logs: a watcher's file is left as it was however often it is sent. +_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO", "OpensearchRemoteLogIO"}) +# Remote log handlers that empty a file they stored and only log a failed write: lines left mean it failed. +_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO", "WasbRemoteLogIO"}) +# Remote log handlers that append to the stored object and keep the file: upload_watcher_log_file empties it +# once stored, as a file sent again would be stored twice. So here too lines left mean a failed write. +_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"}) +_LINES_LEFT_MEAN_FAILED_HANDLERS = _EMPTIES_STORED_FILE_HANDLERS | _APPENDING_HANDLERS +# Remote log handlers that need none of the above. Any handler not named in these lists is logged once (see +# _handler_is): a renamed one would otherwise silently lose what its name was listed for. +# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver is a Cloud Logging subclass). +_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO", "CloudLoggingRemoteLogIO"}) +_KNOWN_HANDLERS = ( + _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS | _APPENDING_HANDLERS | _PLAIN_HANDLERS +) +_checked_handler_classes: set[type] = set() + + +def get_log_permissions() -> tuple[int, int]: + """Return the modes of new log folders and files, as ``init_log_file`` gives them.""" + return ( + int(conf.get("logging", "file_task_handler_new_folder_permissions", fallback="0o775"), 8), + int(conf.get("logging", "file_task_handler_new_file_permissions", fallback="0o664"), 8), + ) + + +def _has_lines(path: Path) -> bool: + try: + return path.stat().st_size > 0 + except OSError: + return False + + +def handler_keeps_no_copy(handler) -> bool: + """ + Whether a file the handler leaves as it was after ``upload`` is no sign of a failed upload. + + True for a handler that streams each line as it is written (it has processors, like CloudWatch) or that takes + only task logs (Elasticsearch, OpenSearch; by class name, as nothing else tells them apart). Any other handler + that leaves the file as it was failed to upload it (S3, GCS and WASB report that only in their log). + """ + if getattr(handler, "processors", None): + return True + return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS) + + +_handler_keeping_local_copy: tuple[object, object] | None = None + + +def upload_watcher_log_file(handler, relative_path: str, path: Path) -> None: + """ + Upload one of a watcher's files, at ``path`` locally; with delete_local_logs, remove it once stored. + + Not by the handler, which removes the whole directory of the file it stored: the watcher's other files are + in it or below it, and may not be stored yet. + """ + delete = getattr(handler, "delete_local_copy", False) and not handler_keeps_no_copy(handler) + if _handler_is(handler, _APPENDING_HANDLERS): + # As OSSRemoteLogIO.upload, which drops oss_write's result: OSS reports a failed write only + # through it. The file is emptied once stored, or removed if it can't be (sent again, it would be + # stored twice), and kept as it was otherwise. + if path.is_file() and handler.oss_write(path.read_text(), relative_path): + try: + os.truncate(path, 0) + except OSError: + path.unlink(missing_ok=True) + else: + (_get_handler_keeping_local_copy(handler) if delete else handler).upload(relative_path, None) + if delete and not _is_left_unstored(handler, path, raised=False): + with suppress(FileNotFoundError): + path.unlink() + with suppress(OSError): + path.parent.rmdir() # once its last file is stored + + +def _get_handler_keeping_local_copy(handler): + """Return a copy of the handler with ``delete_local_copy`` off, made once: it has its own hook.""" + global _handler_keeping_local_copy + if _handler_keeping_local_copy is None or _handler_keeping_local_copy[0] is not handler: + keeping = copy.copy(handler) + keeping.delete_local_copy = False + _handler_keeping_local_copy = (handler, keeping) + return _handler_keeping_local_copy[1] + + +def _append_file(lines: Path, kept: Path) -> None: + """Append ``lines`` to ``kept``; cut back if that fails part way (a full disk), as ``lines`` still has them.""" + size = kept.stat().st_size + try: + with lines.open("rb") as source, kept.open("ab") as target: + shutil.copyfileobj(source, target) + except OSError: + with suppress(OSError): + os.truncate(kept, size) + raise + + +def _move_into_rotated_dir(held: Path, rotated: Path) -> None: + rotated.parent.mkdir(mode=get_log_permissions()[0], parents=True, exist_ok=True) + os.replace(held, rotated) + + +def _is_left_unstored(handler, path: Path, *, raised: bool) -> bool: + """ + Whether to send a watcher's file again after an upload. + + True if it has lines and the upload raised, or S3, GCS, WASB or OSS left them (others may keep them). + """ + if not _has_lines(path): + return False + return raised or _handler_is(handler, _LINES_LEFT_MEAN_FAILED_HANDLERS) + + +def _handler_is(handler, names: frozenset[str]) -> bool: + """Whether the handler's class, or a base class, has one of these names (as for the above).""" + classes = type(handler).__mro__ + if type(handler) not in _checked_handler_classes: + _checked_handler_classes.add(type(handler)) + if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes): + log.info( + "Remote log handler not known by name: asset watcher log files are sent to it as to one " + "that stores them, and sent again only if the upload raises", + handler=f"{type(handler).__module__}.{type(handler).__qualname__}", + ) + return any(cls.__name__ in names for cls in classes) + + +def _get_existing_trigger_ids(session, trigger_ids: set[int]) -> set[int]: + return { + trigger_id + for ids in chunks(sorted(trigger_ids), _TRIGGER_ID_QUERY_CHUNK_SIZE) + for trigger_id in session.scalars(select(Trigger.id).where(Trigger.id.in_(ids))) + } + + +def get_watcher_log_max_open_files() -> int: + """ + Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a quarter of ``ulimit -n``. + + The task and callback trigger logs, the database and the sockets need descriptors too. + """ + setting = conf.getint("triggerer", "watcher_log_max_open_files") + try: + import resource # Unix only + + soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (ImportError, OSError, ValueError): + return setting + if soft == resource.RLIM_INFINITY: + return setting + limit = max(1, soft // 4) + return limit if setting <= 0 else min(setting, limit) + + [email protected](kw_only=True, eq=False) +class _Release: + """The upload of a released watcher's files.""" + + factory: TriggerLoggingFactory + # The upload of the same watcher's files released before, if it was still pending + previous: _Release | None + upload: Future = attrs.field(init=False) + + [email protected](kw_only=True) +class WatcherLogs: + """ + The asset watcher log files of one triggerer's supervisor, other than writing to them. + + Keeps at most ``watcher_log_max_open_files`` of them open, uploads their rotated files and those of + released watchers off the main loop, and uploads or removes the files triggerers that died left here. + """ + + # The supervisor's trigger log factories, by trigger id + logger_cache: dict[int, TriggerLoggingFactory] + # Told when closing a trigger's log file fails + on_error: Callable[[int | None, OSError], None] + # Asset watcher files with an open descriptor, least recently written first + _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False) + watcher_log_max_open_files: int = attrs.field(factory=get_watcher_log_max_open_files, init=False) + # When to look for watcher files left by triggerers that died, and the thread doing it + _orphan_scans_at: list[float] | None = attrs.field(default=None, init=False) + _orphan_scan_thread: threading.Thread | None = attrs.field(default=None, init=False) + # Uploads of released watchers' files, and of rotated files in their own pool (release uploads wait) + _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None, init=False) + _releases: dict[int, _Release] = attrs.field(factory=dict, init=False) + # Held to store a release, and to forget it on its upload's thread + _releases_lock: threading.Lock = attrs.field(factory=threading.Lock, init=False) + # Log paths of watchers that left while their trigger existed, removed here once it no longer does + _left: dict[int, str] = attrs.field(factory=dict, init=False) + _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None, init=False) + _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict, init=False) + # The last number given to each watcher's rotated files here (see upload_rotated) + _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False) + # Each watcher's last rotated file whose upload the handler did not confirm, sent again before it is replaced + _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict, init=False) + # Held while a released watcher's file is added to its held file, or the held file is taken for upload + _held_lock: threading.Lock = attrs.field(factory=threading.Lock, init=False) + + def note_watcher_log_write(self, trigger_id: int) -> None: + """Keep at most ``watcher_log_max_open_files`` watcher files open, closing the least recent.""" + self._open_watcher_logs.pop(trigger_id, None) + self._open_watcher_logs[trigger_id] = None + while 0 < self.watcher_log_max_open_files < len(self._open_watcher_logs): + idle = next(iter(self._open_watcher_logs)) + del self._open_watcher_logs[idle] + if (factory := self.logger_cache.get(idle)) is not None: + factory.close() + + def forget_open_file(self, trigger_id: int) -> None: + """Stop counting a trigger's file as open, as it is about to be closed.""" + self._open_watcher_logs.pop(trigger_id, None) + + def release(self, trigger_id: int, factory: TriggerLoggingFactory) -> None: + """ + Upload and close the file of a watcher no longer assigned here, off the main loop. + + The watcher may come back here at once, on a new live file: this one is held next to it until an upload + of the watcher's files here takes it (see _hold). + """ + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + # In the background: a triggerer that lost its watchers (a DB partition) releases them all at once. + if self._upload_pool is None: + self._upload_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="trigger-log-release") + handler = load_remote_log_handler() + if handler is not None and not _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS): + if hasattr(factory, "bound_logger"): + factory.close() + self._hold(factory.bound_logger._logger._path) + with self._releases_lock: + release = _Release(factory=factory, previous=self._releases.get(trigger_id)) + release.upload = self._upload_pool.submit(self._upload_released, trigger_id, release) + self._releases[trigger_id] = release + + def forget(done: Future) -> None: + with self._releases_lock: + if self._releases.get(trigger_id) is release: + self._releases.pop(trigger_id, None) + + release.upload.add_done_callback(forget) + + def _hold(self, live: Path) -> None: + """Move a released watcher's lines out of the way of a watcher back here (see release).""" + held = live.with_name(TRIGGER_LOG_HELD_FILE_NAME) + # Else left in the live file: a watcher back here appends to it, and its next upload sends them. + with self._held_lock, suppress(OSError): + if not _has_lines(live): + return + if not held.exists(): + os.replace(live, held) + return + # Released again before the held file was taken for upload: one file, sent once. + _append_file(live, held) + live.unlink() + + def _take_held(self, trigger_id: int, log_path: str) -> str | None: + """Move the held file into the rotated directory for upload; return its path there, if there was one.""" + from airflow.sdk.log import load_remote_log_handler # noqa: SDK001 + + base = Path(conf.get("logging", "base_log_folder")) + held = base / get_trigger_log_held_path(log_path) + if (handler := load_remote_log_handler()) is None or not held.is_file(): + return None + if handler_keeps_no_copy(handler): + path = get_trigger_log_rotated_path(log_path) + else: + # Numbered only now, as the watcher's rotations wait for this upload (see prepare_rotation). + part = self._rotated_parts[trigger_id] = self._rotated_parts.get(trigger_id, 0) + 1 Review Comment: Real one: with HDFS each release re-sent every earlier part (15 uploads for 5 releases in a test) and kept them all on disk, so a release now removes the parts it confirmed stored, the same check a rotation uses. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
