vincbeck commented on code in PR #76527:
URL: https://github.com/apache/airflow/pull/76527#discussion_r4602487039
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -564,6 +592,59 @@ def upload_to_remote(self):
upload_to_remote(self.bound_logger, self.ti)
[email protected](kw_only=True)
+class WatcherLoggingFactory(TriggerLoggingFactory):
+ """
+ An asset watcher's own log file on this triggerer, as its owner has no log.
+
+ It is kept while the trigger is assigned here, across restarts of the
watcher (SQS-style ones restart
+ after every event), so it is appended to, and capped on local disk. It is
uploaded with the remote log
+ handler task logs use, and every line keeps its ``trigger_id``.
+ """
+
+ max_bytes: int = 0
+ to_stdout: bool = True
+ # Told about every line, to cap how many of these files are open at once.
+ on_write: Callable[[int], None] | None = attrs.field(default=None,
repr=False)
+ may_rotate: Callable[[], bool] | None = attrs.field(default=None,
repr=False)
+ on_rotate: Callable[[], None] | None = attrs.field(default=None,
repr=False)
+
+ def _open(self, processors: Iterable[structlog.typing.Processor]) -> None:
+ if hasattr(self, "bound_logger"):
+ # Before the processors run: a handler streaming each line
(CloudWatch) names its stream after the
+ # open file, and would drop the line of a file the open-files cap
closed.
+ self.bound_logger._logger.ensure_open()
+ else:
+ from airflow.sdk.log import init_log_file # noqa: SDK001
+
+ underlying_logger = _CappedBytesLogger(
+ init_log_file(self.log_path), self.max_bytes, self.may_rotate,
self.on_rotate
+ )
+ self.bound_logger = structlog.wrap_logger(underlying_logger,
processors=processors).bind(
+ trigger_id=self.trigger_id
+ )
+ if self.on_write is not None and self.trigger_id is not None:
+ self.on_write(self.trigger_id)
+
+ def close(self):
+ if hasattr(self, "bound_logger"):
+ self.bound_logger._logger.close()
+
+ def upload_to_remote(self):
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ if not hasattr(self, "bound_logger") or (handler :=
load_remote_log_handler()) is None:
+ return
+ # Never upload an open file: remote handlers empty or delete it
afterwards. The rotated files first, as
+ # remote handlers append in upload order. Nor an empty one to a
handler that stores files: it would
+ # download and upload the remote copy again for nothing.
+ self.close()
+ live = self.bound_logger._logger._path
+ for path in list_trigger_log_files(live.parent):
+ if _handler_keeps_no_copy(handler) or (path.is_file() and
path.stat().st_size):
Review Comment:
With OSS, the same lines end up in the remote copy more than once.
`OSSRemoteLogIO.upload` appends and never empties the local file, so a
non-empty file here doesn't mean it still needs uploading. Example: a rotated
part is uploaded when it rotates, then sent again here when the watcher leaves.
Same after a graceful stop: the next triggerer on that disk sees the dead job's
files and uploads them again. Fix: for `_APPENDING_HANDLERS`, empty the local
file after an upload that didn't raise.
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -1097,7 +1220,28 @@ def _create_workload(
# log file's parent directory after upload, which must not
hold other callbacks' logs.
self.logger_cache[trigger.id] = TriggerLoggingFactory(
log_path=f"triggerer_callbacks/{dag_id}/{run_id}/{callback.id}/callback.log",
+ trigger_id=trigger.id,
+ on_error=self._warn_trigger_log_error,
)
+ elif trigger.id not in self.logger_cache:
+ may_log, released = self.watcher_logs.take_back(trigger.id)
+ if not may_log:
+ return None # tried again next loop, not to stop the main
loop
Review Comment:
If the remote store is down, this can keep a watcher from running for
minutes just because its old log file is still uploading, and nothing is logged
about it. Can it start on a new file instead of waiting?
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -1097,7 +1220,28 @@ def _create_workload(
# log file's parent directory after upload, which must not
hold other callbacks' logs.
self.logger_cache[trigger.id] = TriggerLoggingFactory(
log_path=f"triggerer_callbacks/{dag_id}/{run_id}/{callback.id}/callback.log",
+ trigger_id=trigger.id,
+ on_error=self._warn_trigger_log_error,
)
+ elif trigger.id not in self.logger_cache:
+ may_log, released = self.watcher_logs.take_back(trigger.id)
+ if not may_log:
+ return None # tried again next loop, not to stop the main
loop
+ if released is not None:
+ self.logger_cache[trigger.id] = released
+ else:
+ # No owner with a log: one file for every instance of the
watcher.
+ log_path = get_trigger_log_relative_path(trigger.id,
self.job.id if self.job else self.id)
Review Comment:
With HDFS and `delete_local_logs`, a watcher that comes back to the same
triggerer loses its earlier lines. `HdfsRemoteLogIO.upload` replaces the remote
file, then deletes the local directory. When the watcher comes back it gets a
new empty `trigger.log` at the same path, and the next release replaces the
remote copy with only the new lines. Fix: upload each assignment's live file to
its own object, like the rotated parts.
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,535 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class _CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_max_bytes", "_may_rotate", "_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
Review Comment:
nit: this directory and the `rotated` one are created with the umask, not
with `file_task_handler_new_folder_permissions` /
`file_task_handler_new_file_permissions` like `init_log_file` does. With
`delete_local_logs` they're recreated after every upload.
##########
airflow-core/src/airflow/jobs/triggerer_job_runner.py:
##########
@@ -142,6 +143,8 @@
from airflow.utils.helpers import prune_dict
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.log.task_log_address import prepare_task_log_contexts,
render_task_log_filename
+from airflow.utils.log.trigger_log import get_trigger_log_relative_path,
list_trigger_log_files
+from airflow.utils.log.watcher_log import WatcherLogs, _CappedBytesLogger,
_handler_keeps_no_copy
Review Comment:
nit: `_CappedBytesLogger` and `_handler_keeps_no_copy` are used outside
`watcher_log.py`, so they should not be private
##########
airflow-core/src/airflow/utils/log/watcher_log.py:
##########
@@ -0,0 +1,535 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""An asset watcher's own log file on the triggerer: the capped file, its
uploads and the orphan scan."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import suppress
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import attrs
+import structlog
+from sqlalchemy import select
+
+from airflow.configuration import conf
+from airflow.models.trigger import Trigger
+from airflow.utils.helpers import chunks
+from airflow.utils.log.trigger_log import (
+ TRIGGER_ID_KEY,
+ TRIGGER_LOG_FILE_NAME,
+ TRIGGER_LOG_ROOT,
+ TRIGGER_LOG_ROTATED_DIR,
+ TRIGGERER_JOB_ID_KEY,
+ get_trigger_log_part_path,
+ get_trigger_log_rotated_path,
+ list_trigger_log_files,
+ parse_trigger_log_dir_id,
+)
+from airflow.utils.session import create_session
+
+if TYPE_CHECKING:
+ from structlog.typing import FilteringBoundLogger
+
+ from airflow.jobs.triggerer_job_runner import TriggerLoggingFactory,
WatcherLoggingFactory
+
+log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)
+
+
+class _CappedBytesLogger(structlog.BytesLogger):
+ """
+ ``BytesLogger`` appending to ``path``, moved to the rotated directory once
it reaches ``max_bytes``.
+
+ ``may_rotate`` can hold a rotation off, ``on_rotate`` is called after one;
``max_bytes <= 0`` is no
+ cap. A closed or removed file (a log cleaner, ``delete_local_logs``, or on
a shared log folder another
+ triggerer's orphan scan) is reopened by the next line.
+ """
+
+ __slots__ = ("_max_bytes", "_may_rotate", "_on_rotate", "_path", "_size")
+
+ def __init__(
+ self,
+ path: Path,
+ max_bytes: int,
+ may_rotate: Callable[[], bool] | None = None,
+ on_rotate: Callable[[], None] | None = None,
+ ):
+ self._path, self._max_bytes = path, max_bytes
+ self._may_rotate, self._on_rotate = may_rotate, on_rotate
+ super().__init__(path.open("ab"))
+ self._size = self._file.tell()
+
+ @property
+ def rotated_path(self) -> Path:
+ return self._path.parent / TRIGGER_LOG_ROTATED_DIR / self._path.name
+
+ def ensure_open(self) -> None:
+ if not self._file.closed and self._unlinked():
+ # Lines written to a removed file are lost.
+ with suppress(OSError):
+ self._file.close()
+ if self._file.closed:
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ self._file = self._path.open("ab")
+ self._write = self._file.write
+ self._flush = self._file.flush
+ self._size = self._file.tell()
+
+ def _unlinked(self) -> bool:
+ try:
+ return os.fstat(self._file.fileno()).st_nlink == 0
+ except (OSError, ValueError):
+ return False
+
+ def msg(self, message: bytes) -> None:
+ with self._lock:
+ try:
+ self.ensure_open()
+ self._write(message + b"\n")
+ self._flush()
+ self._size += len(message) + 1
+ if 0 < self._max_bytes <= self._size and (self._may_rotate is
None or self._may_rotate()):
+ self._file.close()
+ self.rotated_path.parent.mkdir(parents=True, exist_ok=True)
+ with suppress(FileNotFoundError):
+ os.replace(self._path, self.rotated_path)
+ self.ensure_open()
+ if self._on_rotate is not None:
+ try:
+ self._on_rotate()
+ except Exception:
+ # The line is written: the caller, reading the
runner's logs, must not see this.
+ log.exception(
+ "Failed to upload a rotated trigger log",
path=str(self.rotated_path)
+ )
+ except OSError:
+ # Reopened by the next line: a file removed meanwhile (to free
a full disk) frees its space.
+ with suppress(OSError):
+ self._file.close()
+ raise
+
+ def close(self) -> None:
+ # Closing flushes; on a full disk those lines are lost either way.
+ with suppress(OSError):
+ self._file.close()
+
+ log = debug = info = warn = warning = msg
+ fatal = failure = err = error = critical = exception = msg
+
+
+# Marks a dead triggerer's log directory that one triggerer is uploading (see
handle_orphaned_trigger_logs).
+_ORPHAN_UPLOAD_CLAIM = ".uploading"
+_ORPHAN_UPLOAD_CLAIM_MAX_AGE = 600.0
+_TRIGGER_ID_QUERY_CHUNK_SIZE = 500
+# Remote log handlers that take only task logs: a watcher's file is left as it
was however often it is sent.
+_TASK_LOGS_ONLY_HANDLERS = frozenset({"ElasticsearchRemoteLogIO",
"OpensearchRemoteLogIO"})
+# Remote log handlers that empty a file they stored and only log a failed
write: lines left mean it failed.
+_EMPTIES_STORED_FILE_HANDLERS = frozenset({"S3RemoteLogIO", "GCSRemoteLogIO",
"WasbRemoteLogIO"})
+# Remote log handlers that append to the stored object: a file sent again
would be stored twice.
+_APPENDING_HANDLERS = frozenset({"OSSRemoteLogIO"})
+# Remote log handlers that need none of the above. Any handler not named in
these lists is logged once (see
+# _handler_is): a renamed one would otherwise silently lose what its name was
listed for.
+# CloudWatch and Cloud Logging stream each line as it is written (Stackdriver
is a Cloud Logging subclass).
+_PLAIN_HANDLERS = frozenset({"HdfsRemoteLogIO", "CloudWatchRemoteLogIO",
"CloudLoggingRemoteLogIO"})
+_KNOWN_HANDLERS = (
+ _TASK_LOGS_ONLY_HANDLERS | _EMPTIES_STORED_FILE_HANDLERS |
_APPENDING_HANDLERS | _PLAIN_HANDLERS
+)
+_checked_handler_classes: set[type] = set()
+
+
+def _has_lines(path: Path) -> bool:
+ try:
+ return path.stat().st_size > 0
+ except OSError:
+ return False
+
+
+def _handler_keeps_no_copy(handler) -> bool:
+ """
+ Whether a file the handler leaves as it was after ``upload`` is no sign of
a failed upload.
+
+ True for a handler that streams each line as it is written (it has
processors, like CloudWatch) or that takes
+ only task logs (Elasticsearch, OpenSearch; by class name, as nothing else
tells them apart). Any other handler
+ that leaves the file as it was failed to upload it (S3, GCS and WASB
report that only in their log).
+ """
+ if getattr(handler, "processors", None):
+ return True
+ return _handler_is(handler, _TASK_LOGS_ONLY_HANDLERS)
+
+
+def _handler_is(handler, names: frozenset[str]) -> bool:
+ """Whether the handler's class, or a base class, has one of these names
(as for the above)."""
+ classes = type(handler).__mro__
+ if type(handler) not in _checked_handler_classes:
+ _checked_handler_classes.add(type(handler))
+ if not any(cls.__name__ in _KNOWN_HANDLERS for cls in classes):
+ log.info(
+ "Remote log handler not known by name: asset watcher log files
are sent to it as to one "
+ "that stores them, and sent again only if the upload raises",
+
handler=f"{type(handler).__module__}.{type(handler).__qualname__}",
+ )
+ return any(cls.__name__ in names for cls in classes)
+
+
+def get_watcher_log_max_open_files() -> int:
+ """
+ Return ``[triggerer] watcher_log_max_open_files`` (0: no limit), at most a
quarter of ``ulimit -n``.
+
+ The task and callback trigger logs, the database and the sockets need
descriptors too.
+ """
+ setting = conf.getint("triggerer", "watcher_log_max_open_files")
+ try:
+ import resource # Unix only
+
+ soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+ except (ImportError, OSError, ValueError):
+ return setting
+ if soft == resource.RLIM_INFINITY:
+ return setting
+ limit = max(1, soft // 4)
+ return limit if setting <= 0 else min(setting, limit)
+
+
[email protected](kw_only=True)
+class WatcherLogs:
+ """
+ The asset watcher log files of one triggerer's supervisor, other than
writing to them.
+
+ Keeps at most ``watcher_log_max_open_files`` of them open, uploads their
rotated files and those of
+ released watchers off the main loop, and uploads or removes the files
triggerers that died left here.
+ """
+
+ # The supervisor's trigger log factories, by trigger id
+ logger_cache: dict[int, TriggerLoggingFactory]
+ # Told when closing a trigger's log file fails
+ on_error: Callable[[int | None, OSError], None]
+ # Asset watcher files with an open descriptor, least recently written first
+ _open_watcher_logs: dict[int, None] = attrs.field(factory=dict, init=False)
+ watcher_log_max_open_files: int =
attrs.field(factory=get_watcher_log_max_open_files, init=False)
+ # When to look for watcher files left by triggerers that died, and the
thread doing it
+ _orphan_scans_at: list[float] | None = attrs.field(default=None,
init=False)
+ _orphan_scan_thread: threading.Thread | None = attrs.field(default=None,
init=False)
+ # Uploads of released watchers' files, and of rotated files in their own
pool: release uploads wait
+ # for them
+ _upload_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_uploads: dict[int, Future] = attrs.field(factory=dict, init=False)
+ # The released watchers' factories while their upload is pending: one back
before it starts keeps it
+ _released_factories: dict[int, WatcherLoggingFactory] =
attrs.field(factory=dict, init=False)
+ _rotated_pool: ThreadPoolExecutor | None = attrs.field(default=None,
init=False)
+ _pending_rotated_uploads: dict[int, Future] = attrs.field(factory=dict,
init=False)
+ # The last number given to each watcher's rotated files here (see
upload_rotated)
+ _rotated_parts: dict[int, int] = attrs.field(factory=dict, init=False)
+ # Each watcher's last rotated file whose upload the handler did not
confirm, sent again before it is replaced
+ _rotated_unconfirmed: dict[int, str] = attrs.field(factory=dict,
init=False)
+
+ def note_watcher_log_write(self, trigger_id: int) -> None:
+ """Keep at most ``watcher_log_max_open_files`` watcher files open,
closing the least recent."""
+ self._open_watcher_logs.pop(trigger_id, None)
+ self._open_watcher_logs[trigger_id] = None
+ while 0 < self.watcher_log_max_open_files <
len(self._open_watcher_logs):
+ idle = next(iter(self._open_watcher_logs))
+ del self._open_watcher_logs[idle]
+ if (factory := self.logger_cache.get(idle)) is not None:
+ factory.close()
+
+ def forget_open_file(self, trigger_id: int) -> None:
+ """Stop counting a trigger's file as open, as it is about to be
closed."""
+ self._open_watcher_logs.pop(trigger_id, None)
+
+ def release(self, trigger_id: int, factory: WatcherLoggingFactory) -> None:
+ """Upload and close the file of a watcher no longer assigned here, off
the main loop."""
+ # In the background: a triggerer that lost its watchers (a DB
partition) releases them all at once.
+ if self._upload_pool is None:
+ self._upload_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-release")
+ upload = self._upload_pool.submit(self.upload_and_close, trigger_id,
factory)
+ self._pending_uploads[trigger_id] = upload
+ self._released_factories[trigger_id] = factory
+
+ def forget(done: Future) -> None:
+ if self._pending_uploads.get(trigger_id) is done:
+ self._pending_uploads.pop(trigger_id, None)
+ self._released_factories.pop(trigger_id, None)
+
+ upload.add_done_callback(forget)
+
+ def take_back(self, trigger_id: int) -> tuple[bool, WatcherLoggingFactory
| None]:
+ """Return whether a watcher assigned here again may log yet, and its
released factory if it keeps it."""
+ if (upload := self._pending_uploads.get(trigger_id)) is None or
upload.done():
+ return True, None
+ released = self._released_factories.get(trigger_id)
+ if released is None or not upload.cancel():
+ # Back while its file uploads: the handler empties or removes it
once uploaded, which would lose
+ # the new lines.
+ return False, None
+ # Its upload had not started (queued behind others, minutes each with
an unreachable store): keep the
+ # same file, uploaded when the watcher leaves again or the triggerer
stops.
+ return True, released
+
+ def rotated_upload_done(self, trigger_id: int) -> bool:
+ upload = self._pending_rotated_uploads.get(trigger_id)
+ if upload is not None and not upload.done():
+ return False
+ if self._rotated_pool is not None and (path :=
self._rotated_unconfirmed.pop(trigger_id, None)):
+ # Now, while it is the only file in its directory: with
delete_local_logs, a handler deletes the
+ # directory of the file it uploaded. Replaced by the next rotation
whether it goes through or not.
+ self._pending_rotated_uploads[trigger_id] =
self._rotated_pool.submit(
+ self._upload_rotated_again, trigger_id, path
+ )
+ return False
+ return True
+
+ @staticmethod
+ def _upload_rotated_again(trigger_id: int, path: str) -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ try:
+ if (handler := load_remote_log_handler()) is not None:
+ handler.upload(path, None)
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+
+ def upload_rotated(self, trigger_id: int, log_path: str) -> None:
+ # Called by the line that rotated the file: the rest runs in the pool,
not to hold up the runner's logs.
+ if self._rotated_pool is None:
+ self._rotated_pool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="trigger-log-rotated")
+ rotated = get_trigger_log_rotated_path(log_path)
+ # Numbered here: a watcher's next rotation waits for this upload (see
rotated_upload_done).
+ part = self._rotated_parts.get(trigger_id, 0) + 1
+
+ def upload() -> None:
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ path = rotated
+ base = Path(conf.get("logging", "base_log_folder"))
+ try:
+ if (handler := load_remote_log_handler()) is None:
+ return
+ if not _handler_keeps_no_copy(handler):
+ # A handler that stores files gets each rotated file as
its own object (see
+ # get_trigger_log_part_path), numbered by this triggerer,
the only one writing there.
+ # Counted, not read from the disk: a watcher that comes
back gets a new file, and the parts
+ # it uploaded before are gone locally.
+ for done in (base / rotated).parent.glob("*.log"):
+ if done.name != TRIGGER_LOG_FILE_NAME:
+ with suppress(OSError):
+ done.unlink() # the previous one, uploaded or
tried twice: one at a time
+ path = get_trigger_log_part_path(log_path, part)
+ # Named only now: until then the triggerer's log server
serves it as the rotated file.
+ os.replace(base / rotated, base / path)
+ handler.upload(path, None)
+ raised = False
+ except Exception:
+ log.exception("Failed to upload a rotated trigger log",
trigger_id=trigger_id, path=path)
+ raised = True
+ if path != rotated and _has_lines(base / path):
+ # Sent again if the upload raised, or if S3, GCS or WASB left
lines (others may keep them).
+ if (raised and not _handler_is(handler, _APPENDING_HANDLERS))
or (
+ not raised and _handler_is(handler,
_EMPTIES_STORED_FILE_HANDLERS)
+ ):
+ self._rotated_unconfirmed[trigger_id] = path
+
+ self._pending_rotated_uploads[trigger_id] =
self._rotated_pool.submit(upload)
+ self._rotated_parts[trigger_id] = part
+
+ def upload_and_close(self, trigger_id: int, factory:
TriggerLoggingFactory) -> None:
+ if (rotated_upload := self._pending_rotated_uploads.pop(trigger_id,
None)) is not None:
+ rotated_upload.result() # else the rotated file could be read
again before it is emptied
+ self._rotated_unconfirmed.pop(trigger_id, None) # sent again below,
with the live file
+ try:
+ factory.upload_to_remote()
+ except Exception:
+ log.exception("Failed to upload trigger logs to remote",
trigger_id=trigger_id)
+ finally:
+ # Close even if the upload failed, not to leak the descriptor.
Closing flushes, which can fail.
+ try:
+ factory.close()
+ except OSError as e:
+ self.on_error(trigger_id, e)
+
+ def upload_and_close_all(self, factories: list[tuple[int,
TriggerLoggingFactory]]) -> None:
+ """Upload and close these watchers' files, and wait for the released
ones' uploads."""
+ self._open_watcher_logs.clear()
+ # Side by side: one after the other, thousands of files take longer
than a container's stop timeout.
+ with ThreadPoolExecutor(max_workers=16,
thread_name_prefix="trigger-log-upload") as pool:
+ list(pool.map(lambda item: self.upload_and_close(*item),
factories))
+ if self._upload_pool is not None:
+ self._upload_pool.shutdown(wait=True)
+
+ def handle_orphaned_trigger_logs(self, own_job_id: int | None) -> None:
+ """
+ Upload, or remove, the asset watcher files left on this disk by
triggerers that are no longer alive.
+
+ A killed triggerer uploads nothing, so its files are uploaded now,
then their directories removed.
+ Files remote logging doesn't take (it is off, or the handler keeps no
copy, like Elasticsearch) are
+ removed instead: all but the most recently written for each watcher,
and all for triggers that no
+ longer exist. Files of live triggerers, this one included, of
triggerers without a job (no job id
+ directory), and those a failed upload left stay.
+ """
+ from airflow.jobs.job import Job
+ from airflow.sdk.log import load_remote_log_handler # noqa: SDK001
+
+ handler = load_remote_log_handler()
+ base = Path(conf.get("logging", "base_log_folder"))
+ dirs_by_job: dict[int, list[Path]] = {}
+ dirs_by_trigger: dict[Path, int] = {}
+ for producer_dir in
base.glob(f"{TRIGGER_LOG_ROOT}/{TRIGGER_ID_KEY}=*/{TRIGGERER_JOB_ID_KEY}=*"):
+ # Only directories this module names are ever removed: both ids
must parse.
+ trigger_id = parse_trigger_log_dir_id(TRIGGER_ID_KEY,
producer_dir.parent.name)
+ job_id = parse_trigger_log_dir_id(TRIGGERER_JOB_ID_KEY,
producer_dir.name)
+ if (
+ trigger_id is not None
+ and job_id is not None
+ and job_id != own_job_id
Review Comment:
What removes the files of watchers that left this triggerer while it keeps
running? The scan skips its own job id and only runs at start, so without
remote logging the files of deleted triggers (up to twice
`watcher_log_max_bytes` each) stay until the triggerer dies.
##########
airflow-core/tests/unit/jobs/test_triggerer_job.py:
##########
@@ -1545,6 +1560,1233 @@ def
test_trigger_logger_fd_closed_when_upload_to_remote_raises(jobless_superviso
assert 42 not in jobless_supervisor.running_triggers
+class TestWatcherLogFile:
+ @staticmethod
+ def _watcher_trigger(mocker, trigger_id):
+ asset = mocker.Mock(uri="s3://bucket/orders")
+ asset.name = "orders"
+ watcher = mocker.Mock(asset=asset)
+ watcher.name = "new_files"
+ return mocker.Mock(
+ spec=Trigger,
+ id=trigger_id,
+ classpath="some.path.Trigger",
+ encrypted_kwargs="",
+ task_instance=None,
+ callback=None,
+ assets=[asset],
+ asset_watchers=[watcher],
+ )
+
+ @staticmethod
+ def _create(supervisor, trigger, mocker):
+ supervisor._create_workload(
+ trigger=trigger, dag_bag=mocker.Mock(),
render_log_fname=mocker.Mock(), session=mocker.Mock()
+ )
+
+ @conf_vars({("triggerer", "watcher_log_max_bytes"): "1234"})
+ def test_one_file_per_assignment_and_triggerer(self, jobless_supervisor,
mocker):
+ trigger = self._watcher_trigger(mocker, 42)
+
+ self._create(jobless_supervisor, trigger, mocker)
+ factory = jobless_supervisor.logger_cache[42]
+ # A restarted instance (e.g. an SQS-style watcher after each event)
keeps writing the same file.
+ self._create(jobless_supervisor, trigger, mocker)
+
+ assert isinstance(factory, WatcherLoggingFactory)
+ assert (
+ factory.log_path ==
f"triggers/trigger_id=42/triggerer_job_id={jobless_supervisor.id}/trigger.log"
+ )
+ assert (factory.trigger_id, factory.max_bytes) == (42, 1234)
+ assert jobless_supervisor.logger_cache[42] is factory
+
+ def test_a_callback_trigger_logs_to_the_callback_log_only(self,
jobless_supervisor, mocker):
+ trigger = self._watcher_trigger(mocker, 42)
+ trigger.assets = []
+ trigger.callback = mocker.Mock(data={"dag_id": "d"}, id=uuid.uuid4())
+
+ self._create(jobless_supervisor, trigger, mocker)
+
+ factory = jobless_supervisor.logger_cache[42]
+ assert not isinstance(factory, WatcherLoggingFactory)
+ assert factory.log_path ==
f"triggerer_callbacks/d/unknown/{trigger.callback.id}/callback.log"
+
+ @pytest.mark.parametrize(
+ "changes",
+ [
+ pytest.param(
+ messages.TriggerStateChanges(events=None, finished=[42],
running_ids=set()), id="finished"
+ ),
+ pytest.param(
+ messages.TriggerStateChanges(events=None, running_ids=set()),
id="lost-by-the-runner"
+ ),
+ ],
+ )
+ def test_file_is_kept_across_instance_restarts(self, jobless_supervisor,
mocker, changes):
+ factory = mocker.MagicMock(spec=WatcherLoggingFactory)
+ jobless_supervisor.logger_cache[42] = factory
+ jobless_supervisor.running_triggers.add(42)
+ mocker.patch.object(TriggerRunnerSupervisor,
"build_trigger_workloads", return_value=[])
+
+ jobless_supervisor._handle_request(changes,
log=mocker.Mock(spec=FilteringBoundLogger), req_id=0)
+ jobless_supervisor.update_triggers({42})
+
+ factory.close.assert_not_called()
+ assert jobless_supervisor.logger_cache[42] is factory
+
+ @pytest.mark.parametrize(
+ ("assignments", "queue"),
+ [
+ pytest.param({42: 7}, "releasing_triggers", id="moved"),
+ pytest.param({}, "cancelling_triggers", id="gone"),
+ ],
+ )
+ def test_file_is_closed_one_loop_after_the_trigger_left(
+ self, jobless_supervisor, mocker, assignments, queue
+ ):
+ factory = mocker.MagicMock(spec=WatcherLoggingFactory)
+ jobless_supervisor.logger_cache[42] = factory
+ jobless_supervisor.running_triggers.add(42)
+ mocker.patch.object(TriggerRunnerSupervisor,
"fetch_trigger_assignments", return_value=assignments)
+ log = mocker.Mock(spec=FilteringBoundLogger)
+
+ jobless_supervisor.update_triggers(set())
+ assert 42 in getattr(jobless_supervisor, queue)
+ # Still open: the instance's last lines (on_kill, cleanup) are still
to come on the log socket.
+
jobless_supervisor._handle_request(messages.TriggerStateChanges(finished=[42]),
log=log, req_id=0)
+ factory.close.assert_not_called()
+
+ jobless_supervisor.update_triggers(set())
+ jobless_supervisor.watcher_logs._upload_pool.shutdown(wait=True)
+
+ factory.upload_to_remote.assert_called_once()
+ factory.close.assert_called()
+ assert 42 not in jobless_supervisor.logger_cache
+
+ def test_close_trigger_log_files_uploads_only_watcher_logs(self,
jobless_supervisor, mocker):
+ own_file = mocker.MagicMock(spec=WatcherLoggingFactory)
+ task_log = mocker.MagicMock(spec=TriggerLoggingFactory)
+ jobless_supervisor.logger_cache.update({2: own_file, 3: task_log})
+
+ jobless_supervisor.close_trigger_log_files()
+
+ own_file.upload_to_remote.assert_called_once()
+ task_log.upload_to_remote.assert_not_called()
+ assert jobless_supervisor.logger_cache == {3: task_log}
+
+ @staticmethod
+ def _processors():
+ return [JSONRenderer(serializer=lambda obj, **_:
json.dumps(obj).encode())]
+
+ @staticmethod
+ def _factory(max_bytes=0, **kwargs):
+ return WatcherLoggingFactory(
+ log_path=get_trigger_log_relative_path(1, 5),
+ trigger_id=1,
+ max_bytes=max_bytes,
+ to_stdout=False,
+ **kwargs,
+ )
+
+ @pytest.mark.parametrize("max_bytes", [0, 400])
+ def test_file_is_appended_to_when_reopened(self, tmp_path, max_bytes):
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ for run in range(2):
+ factory = self._factory(max_bytes)
+ factory(self._processors()).log(logging.INFO, "line", run=run)
+ factory.close()
+
+ lines = (tmp_path / get_trigger_log_relative_path(1,
5)).read_text().splitlines()
+ assert [json.loads(line)["run"] for line in lines] == [0, 1]
+ assert {json.loads(line)["trigger_id"] for line in lines} == {1}
+
+ def test_a_closed_file_is_reopened_by_the_next_line(self, tmp_path):
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory()
+ factory(self._processors()).log(logging.INFO, "line", n=1)
+ factory.close()
+ factory(self._processors()).log(logging.INFO, "line", n=2)
+ factory.close()
+
+ lines = (tmp_path / get_trigger_log_relative_path(1,
5)).read_text().splitlines()
+ assert [json.loads(line)["n"] for line in lines] == [1, 2]
+
+ def
test_a_closed_file_is_open_again_before_the_processors_see_the_next_line(self,
tmp_path):
+ # A remote handler that streams each line (CloudWatch) names its
stream after the open file.
+ seen_closed = []
+
+ def record(logger, method_name, event_dict):
+ seen_closed.append(logger._file.closed)
+ return event_dict
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory()
+ factory([record, *self._processors()]).log(logging.INFO, "line",
n=1)
+ factory.close()
+ factory([record, *self._processors()]).log(logging.INFO, "line",
n=2)
+ factory.close()
+
+ assert seen_closed == [False, False]
+
+ @pytest.mark.parametrize("max_bytes", [0, 80], ids=["no-cap", "rotates"])
+ @pytest.mark.parametrize("closed", [True, False], ids=["closed", "open"])
+ def test_a_removed_directory_is_recreated_by_the_next_line(self, tmp_path,
max_bytes, closed):
+ # Open: on a shared log folder, another triggerer's orphan scan can
remove a running one's files.
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory(max_bytes)
+ logger = factory(self._processors())
+ logger.log(logging.INFO, "line", n=1)
+ if closed:
+ factory.close()
+ shutil.rmtree(tmp_path / get_trigger_log_dir(1))
+ logger.log(logging.INFO, "line", n=2)
+ logger.log(logging.INFO, "line", n=3)
+ factory.close()
+
+ files = [
+ tmp_path / f"{get_trigger_log_dir(1, 5)}/rotated/trigger.log",
+ tmp_path / get_trigger_log_relative_path(1, 5),
+ ]
+ lines = [line for f in files if f.exists() for line in
f.read_text().splitlines()]
+ assert [json.loads(line)["n"] for line in lines] == [2, 3]
+
+ def
test_local_disk_is_capped_with_one_rotated_file_in_its_own_directory(self,
tmp_path):
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory(1000)
+ logger = factory(self._processors())
+ for n in range(500):
+ logger.log(logging.INFO, "line", n=n)
+ factory.close()
+
+ live = tmp_path / get_trigger_log_relative_path(1, 5)
+ rotated = tmp_path / f"{get_trigger_log_dir(1, 5)}/rotated/trigger.log"
+ numbers = [json.loads(line)["n"] for p in (rotated, live) for line in
p.read_text().splitlines()]
+ assert live.stat().st_size < 1000
+ assert rotated.stat().st_size < 1000 + 100
+ assert numbers == list(range(numbers[0], 500))
+
+ def
test_upload_closes_first_and_sends_the_rotated_file_before_the_live_file(self,
tmp_path, mocker):
+ handler = mocker.Mock(processors=()) # stores files, not streaming
each line
+ open_at_upload = []
+ handler.upload.side_effect = lambda path, ti: open_at_upload.append(
+ not factory.bound_logger._logger._file.closed
+ )
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=handler)
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory(200)
+ logger = factory(self._processors())
+ for n in itertools.count():
+ logger.log(logging.INFO, "line", n=n)
+ if (tmp_path / f"{get_trigger_log_dir(1,
5)}/rotated/trigger.log").exists():
+ break
+ logger.log(logging.INFO, "line", n=n + 1) # a store gets only
files with lines
+ factory.upload_to_remote()
+
+ assert [c.args for c in handler.upload.call_args_list] == [
+ (f"{get_trigger_log_dir(1, 5)}/rotated/trigger.log", None),
+ (get_trigger_log_relative_path(1, 5), None),
+ ]
+ assert open_at_upload == [False, False]
+
+ def test_with_remote_logging_every_rotated_file_is_uploaded(self,
jobless_supervisor, mocker, tmp_path):
+ # A watcher stays on one triggerer for days: a rotated file replaced
before it is uploaded is lost.
+ store = self.S3RemoteLogIO(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "200"}
+ ):
+ self._create(jobless_supervisor, self._watcher_trigger(mocker,
42), mocker)
+ factory = jobless_supervisor.logger_cache.pop(42)
+ logger = factory(self._processors())
+ for n in range(50):
+ logger.log(logging.INFO, "line", n=n)
+ jobless_supervisor.watcher_logs.upload_and_close(42, factory)
+
+ assert store.lines() == list(range(50))
+
+ class _Store:
+ """
+ A remote handler that stores files: by default as S3, GCS and WASB do
(appending downloads the object and
+ uploads all of it again, then the local file is emptied), with
``replace`` as HDFS does (the object is
+ replaced, the local file kept), with ``keep`` as OSS does (appended
to, the local file kept).
+ """
+
+ processors = ()
+
+ def __init__(self, base, *, replace=False, keep=False):
+ self.base, self.replace, self.keep, self.down, self.attempts =
base, replace, keep, False, 0
+ self.quiet_failures = (
+ 0 # uploads that leave the file as it was, as S3, GCS and
WASB on a failed write
+ )
+ self.raised_failures = 0 # uploads that raise, as HDFS on a
failed write
+ self.objects: dict[str, str] = {}
+ self.uploads: list[tuple[str, int, int]] = [] # path, bytes of
the file, bytes downloaded
+ self.replaced: list[str] = [] # objects whose content was
replaced by other content
+
+ def upload(self, path, ti):
+ self.attempts += 1
+ if self.down:
+ raise OSError("store unreachable")
+ if self.raised_failures:
+ self.raised_failures -= 1
+ raise OSError("store unreachable")
+ if self.quiet_failures:
+ self.quiet_failures -= 1
+ return
+ local = self.base / path
+ if not local.is_file():
+ return
+ new = local.read_text()
+ old = self.objects.get(path, "")
+ self.uploads.append((path, len(new), 0 if self.replace else
len(old)))
+ if self.replace:
+ if old and old != new:
+ self.replaced.append(path)
+ self.objects[path] = new
+ else:
+ self.objects[path] = old + new
+ if not self.keep:
+ local.write_text("")
+
+ def lines(self):
+ return [
+ json.loads(line)["n"]
+ for key in sorted(self.objects)
+ for line in self.objects[key].splitlines()
+ ]
+
+ class S3RemoteLogIO(_Store):
+ pass
+
+ class HdfsRemoteLogIO(_Store):
+ def __init__(self, base):
+ super().__init__(base, replace=True)
+
+ class CustomRemoteLogIO(_Store):
+ def __init__(self, base):
+ super().__init__(base, keep=True)
+
+ class OSSRemoteLogIO(_Store):
+ def __init__(self, base):
+ super().__init__(base, keep=True)
+
+ def _log(self, supervisor, trigger_id, numbers):
+ logger = supervisor.logger_cache[trigger_id](self._processors())
+ for n in numbers:
+ logger.log(logging.INFO, "line", n=n)
+ if (upload :=
supervisor.watcher_logs._pending_rotated_uploads.get(trigger_id)) is not None:
+ upload.result() # each rotation once the file is full, not
held off by the previous upload
+
+ def _log_until_rotated(self, supervisor, uploads, start=0, trigger_id=42):
+ """Log until the next rotated file is uploaded (``uploads`` counts the
uploads); return the last number."""
+ before = uploads()
+ for n in itertools.count(start):
+ self._log(supervisor, trigger_id, [n])
+ if uploads() > before:
+ return n
+
+ def _run(self, supervisor, store, mocker, tmp_path, numbers=None,
trigger_id=42):
+ """Log ``numbers`` (or until the first rotation) on a new watcher
file, and release it."""
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(supervisor, self._watcher_trigger(mocker,
trigger_id), mocker)
+ if numbers is None:
+ self._log_until_rotated(supervisor, lambda: store.attempts,
trigger_id=trigger_id)
+ else:
+ self._log(supervisor, trigger_id, numbers)
+ supervisor.watcher_logs.upload_and_close(trigger_id,
supervisor.logger_cache.pop(trigger_id))
+
+ def test_each_rotated_file_is_its_own_remote_object(self,
jobless_supervisor, mocker, tmp_path):
+ store = self._Store(tmp_path)
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(200))
+
+ job_dir = get_trigger_log_dir(42, jobless_supervisor.id)
+ parts = [path for path, _, _ in store.uploads if path !=
f"{job_dir}/trigger.log"]
+ assert len(parts) >= 8
+ assert parts == [f"{job_dir}/rotated/trigger.{n:06d}.log" for n in
range(1, len(parts) + 1)]
+ assert store.lines() == list(range(200))
+
+ def test_an_appending_store_never_downloads_earlier_parts(self,
jobless_supervisor, mocker, tmp_path):
+ # S3, GCS and WASB append by downloading the object: to one object,
each rotation would move all before it.
+ store = self._Store(tmp_path)
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(200))
+
+ assert [downloaded for _, _, downloaded in store.uploads] == [0] *
len(store.uploads)
+ assert max(size for _, size, _ in store.uploads) < 400 + 100
+
+ def test_a_replacing_store_loses_no_lines(self, jobless_supervisor,
mocker, tmp_path):
+ # HDFS replaces the object: to one object, each rotation would replace
the one before it.
+ store = self.HdfsRemoteLogIO(tmp_path)
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(200))
+
+ assert store.replaced == []
+ assert store.lines() == list(range(200))
+
+ def
test_a_rotated_file_whose_upload_failed_is_sent_again_before_it_is_replaced(
+ self, jobless_supervisor, mocker, tmp_path
+ ):
+ store = self.S3RemoteLogIO(tmp_path)
+ store.quiet_failures = 1
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(200))
+
+ parts = [path for path, _, _ in store.uploads if "/rotated/" in path]
+ assert store.attempts == len(store.uploads) + 1 # the failed one sent
once more, no other
+ assert len(parts) == len(set(parts))
+ assert store.lines() == list(range(200))
+
+ def test_a_rotated_file_whose_upload_raised_is_sent_again(self,
jobless_supervisor, mocker, tmp_path):
+ # HDFS raises on a failed write, and replaces the object: sent again,
the part is stored once.
+ store = self.HdfsRemoteLogIO(tmp_path)
+ store.raised_failures = 1
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(200))
+
+ assert store.attempts == len(store.uploads) + 1
+ assert store.replaced == []
+ assert store.lines() == list(range(200))
+
+ @pytest.mark.parametrize("store_class", ["HdfsRemoteLogIO",
"OSSRemoteLogIO", "CustomRemoteLogIO"])
+ def test_a_store_that_keeps_the_file_gets_each_part_once(
+ self, jobless_supervisor, mocker, tmp_path, store_class
+ ):
+ # Only S3, GCS and WASB empty a file they stored: lines another store
left are no failed upload.
+ # OSS and this custom store append, so sending it again would store it
twice.
+ store = getattr(self, store_class)(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(jobless_supervisor, self._watcher_trigger(mocker,
42), mocker)
+ self._log(jobless_supervisor, 42, range(200))
+
+ parts = [path for path, _, _ in store.uploads]
+ assert len(parts) >= 8
+ assert len(parts) == len(set(parts))
+ assert len(store.lines()) == len(set(store.lines()))
+
+ def test_a_store_that_stays_down_keeps_one_rotated_file(self,
jobless_supervisor, mocker, tmp_path):
+ store = self._Store(tmp_path)
+ store.down = True
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(jobless_supervisor, self._watcher_trigger(mocker,
42), mocker)
+ self._log(jobless_supervisor, 42, range(200))
+
+ rotated = list((tmp_path / f"{get_trigger_log_dir(42,
jobless_supervisor.id)}/rotated").iterdir())
+ assert len(rotated) == 1
+ assert rotated[0].stat().st_size < 400 + 100
+
+ def test_empty_files_are_not_uploaded_to_a_store(self, jobless_supervisor,
mocker, tmp_path):
+ # Released right after a rotation: the rotated file was emptied by its
upload, the live file is empty.
+ store = self._Store(tmp_path)
+ self._run(jobless_supervisor, store, mocker, tmp_path)
+
+ assert [(path.rsplit("/", 1)[-1], bool(size)) for path, size, _ in
store.uploads] == [
+ ("trigger.000001.log", True)
+ ]
+
+ def test_a_watcher_back_on_the_same_triggerer_does_not_overwrite_its_parts(
+ self, jobless_supervisor, mocker, tmp_path
+ ):
+ # Its new file starts empty, and the parts it uploaded are gone from
the disk, as delete_local_logs does.
+ store = self.HdfsRemoteLogIO(tmp_path)
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(100))
+ first = {path for path, _, _ in store.uploads if "/rotated/" in path}
+ shutil.rmtree(tmp_path / get_trigger_log_dir(42))
+ self._run(jobless_supervisor, store, mocker, tmp_path, range(100, 200))
+ second = {path for path, _, _ in store.uploads if "/rotated/" in path}
- first
+
+ assert first
+ assert second
+ assert max(first) < min(second)
+ assert [path for path in store.replaced if "/rotated/" in path] == []
+
+ def test_an_orphan_pickup_does_not_overwrite_earlier_parts(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ now = timezone.utcnow()
+ dead_job = Job(
+ job_type="TriggererJob", state="running", latest_heartbeat=now -
datetime.timedelta(hours=1)
+ )
+ session.add(dead_job)
+ session.flush()
+ dead = supervisor_builder(job=dead_job)
+ store = self.HdfsRemoteLogIO(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(dead, self._watcher_trigger(mocker, 42), mocker)
+ self._log(dead, 42, range(100))
+ store.down = True # then killed, its last rotated file not
uploaded
+ last = self._log_until_rotated(dead, lambda: store.attempts,
start=100)
+ self._log(dead, 42, range(last + 1, last + 4))
+ store.down = False
+ own = supervisor_builder()
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert store.replaced == []
+ assert store.lines() == list(range(last + 4))
+
+ def test_a_streaming_handler_gets_the_same_uploads_as_before(self,
jobless_supervisor, mocker, tmp_path):
+ # CloudWatch and Cloud Logging send each line as it is logged; upload
only flushes.
+ handler = mocker.Mock(processors=(object(),))
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=handler)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(jobless_supervisor, self._watcher_trigger(mocker,
42), mocker)
+ self._log_until_rotated(jobless_supervisor, lambda:
handler.upload.call_count)
+ jobless_supervisor.watcher_logs.upload_and_close(42,
jobless_supervisor.logger_cache.pop(42))
+
+ job_dir = get_trigger_log_dir(42, jobless_supervisor.id)
+ assert [c.args[0] for c in handler.upload.call_args_list] == [
+ f"{job_dir}/rotated/trigger.log",
+ f"{job_dir}/rotated/trigger.log",
+ f"{job_dir}/trigger.log",
+ ]
+
+ def test_a_rotation_that_fails_leaves_the_line_written(self, tmp_path,
mocker):
+ # E.g. no thread left for the pool: raised out of the line, it stops
the reading of the runner's logs.
+ watcher_logger = mocker.patch("airflow.utils.log.watcher_log.log")
+ triggerer_log = mocker.patch("airflow.jobs.triggerer_job_runner.log")
+ on_rotate = mocker.Mock(side_effect=RuntimeError("can't start new
thread"))
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = self._factory(200, on_rotate=on_rotate)
+ logger = factory(self._processors())
+ n = 0
+ while not on_rotate.called:
+ logger.log(logging.INFO, "line", n=n)
+ n += 1
+ logger.log(logging.INFO, "line", n=n)
+ factory.close()
+
+ files = [
+ tmp_path / f"{get_trigger_log_dir(1, 5)}/rotated/trigger.log",
+ tmp_path / get_trigger_log_relative_path(1, 5),
+ ]
+ numbers = [json.loads(line)["n"] for f in files for line in
f.read_text().splitlines()]
+ assert numbers == list(range(n + 1))
+ assert on_rotate.call_count == 1
+ assert watcher_logger.exception.call_count == 1
+ triggerer_log.log.assert_not_called() # not sent to the triggerer's
output as a line the file lost
+
+ def test_a_rotation_only_queues_its_upload(self, jobless_supervisor,
mocker, tmp_path):
+ # The line that rotated the file was read from the runner's log
socket: nothing more is done there.
+ store = self._Store(tmp_path)
+ threads = []
+
+ def load_remote_log_handler():
+ threads.append(threading.current_thread().name)
+ return store
+
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
side_effect=load_remote_log_handler)
+ with conf_vars(
+ {("logging", "base_log_folder"): str(tmp_path), ("triggerer",
"watcher_log_max_bytes"): "400"}
+ ):
+ self._create(jobless_supervisor, self._watcher_trigger(mocker,
42), mocker)
+ self._log(jobless_supervisor, 42, range(100))
+
+ assert len(threads) >= 4
+ assert all(name.startswith("trigger-log-rotated") for name in threads)
+ job_dir = get_trigger_log_dir(42, jobless_supervisor.id)
+ assert [path for path, _, _ in store.uploads] == [
+ f"{job_dir}/rotated/trigger.{n:06d}.log" for n in range(1,
len(threads) + 1)
+ ]
+
+ def test_at_most_max_open_files_stay_open(self, jobless_supervisor,
mocker, tmp_path):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ jobless_supervisor.watcher_logs.watcher_log_max_open_files = 2
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ for trigger_id in (1, 2, 3):
+ jobless_supervisor.logger_cache[trigger_id] =
WatcherLoggingFactory(
+ log_path=get_trigger_log_relative_path(trigger_id, 5),
+ trigger_id=trigger_id,
+ to_stdout=False,
+
on_write=jobless_supervisor.watcher_logs.note_watcher_log_write,
+ )
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ # 1 is written again before 3 arrives, so 2 is the least recently
written.
+ for trigger_id in (1, 2, 1, 3):
+ gen.send(
+ msgspec.json.encode(
+ {"event": f"from {trigger_id}", "level": "info",
"trigger_id": trigger_id}
+ )
+ )
+
+ def is_open(trigger_id):
+ return not
jobless_supervisor.logger_cache[trigger_id].bound_logger._logger._file.closed
+
+ assert [is_open(t) for t in (1, 2, 3)] == [True, False, True]
+ assert list(jobless_supervisor.watcher_logs._open_watcher_logs) ==
[1, 3]
+ jobless_supervisor.close_trigger_log_files()
+
+ assert [
+ [
+ json.loads(line)["event"]
+ for line in (tmp_path / get_trigger_log_relative_path(t,
5)).read_text().splitlines()
+ ]
+ for t in (1, 2, 3)
+ ] == [["from 1", "from 1"], ["from 2"], ["from 3"]]
+ assert jobless_supervisor.watcher_logs._open_watcher_logs == {}
+
+ @pytest.mark.parametrize("to_stdout", [True, False])
+ def test_watcher_lines_go_to_their_file_and_to_stdout_with_trigger_id(
+ self, jobless_supervisor, mocker, tmp_path, cap_structlog, to_stdout
+ ):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ jobless_supervisor.logger_cache[5] = WatcherLoggingFactory(
+ log_path=get_trigger_log_relative_path(5, 1), trigger_id=5,
to_stdout=to_stdout
+ )
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ gen.send(msgspec.json.encode({"event": "poll 1", "level": "info",
"trigger_id": 5}))
+ jobless_supervisor.close_trigger_log_files()
+
+ line = json.loads((tmp_path / get_trigger_log_relative_path(5,
1)).read_text())
+ assert (line["event"], line["trigger_id"]) == ("poll 1", 5)
+ assert ({"event": "poll 1", "trigger_id": 5} in cap_structlog) is
to_stdout
+
+ def test_task_and_callback_trigger_lines_go_to_their_owners_log_only(
+ self, jobless_supervisor, mocker, cap_structlog
+ ):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ mocker.patch("airflow.sdk.log.logging_processors")
+ task_logger = mocker.Mock()
+ jobless_supervisor.logger_cache[5] = mocker.MagicMock(
+ spec=TriggerLoggingFactory, return_value=task_logger
+ )
+
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ gen.send(msgspec.json.encode({"event": "sleeping", "level": "info",
"trigger_id": 5}))
+
+ task_logger.log.assert_called_once_with(20, "sleeping")
+ assert "sleeping" not in [e["event"] for e in cap_structlog]
+
+ def test_execute_closes_trigger_files_after_the_runner_exits(self, mocker,
session):
+ supervisor = mocker.MagicMock(spec=TriggerRunnerSupervisor,
_exit_code=0)
+ mocker.patch.object(TriggerRunnerSupervisor, "start",
return_value=supervisor)
+ mocker.patch("airflow.jobs.triggerer_job_runner.stats.initialize")
+ job = Job()
+ session.add(job)
+ session.flush()
+ job_runner = TriggererJobRunner(job)
+ mocker.patch.object(job_runner, "register_signals")
+
+ job_runner._execute()
+
+ assert [c[0] for c in supervisor.method_calls if c[0] in ("kill",
"close_trigger_log_files")] == [
+ "kill",
+ "close_trigger_log_files",
+ ]
+
+ def test_orphaned_files_of_triggerers_that_are_not_alive_are_uploaded(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ alive = Job(job_type="TriggererJob", state="running",
latest_heartbeat=now)
+ dead = Job(
+ job_type="TriggererJob", state="running", latest_heartbeat=now -
datetime.timedelta(hours=1)
+ )
+ session.add_all([alive, dead])
+ session.flush()
+ gone = dead.id + 100 # deleted from the job table, e.g. by ``airflow
db clean``
+
+ def write(path, text="line\n"):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text(text)
+
+ write(f"{get_trigger_log_dir(1, dead.id)}/rotated/trigger.log")
+ write(get_trigger_log_relative_path(1, dead.id))
+ write(get_trigger_log_relative_path(2, dead.id), "") # already
uploaded and emptied
+ write(get_trigger_log_relative_path(3, gone))
+ write(get_trigger_log_relative_path(1, alive.id))
+ write(get_trigger_log_relative_path(1, own.job.id))
+ write(get_trigger_log_relative_path(1, uuid.uuid4())) # a triggerer
without a job
+ store = self.S3RemoteLogIO(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ uploads = [path for path, _, _ in store.uploads]
+ assert sorted(uploads) == sorted(
+ [
+ f"{get_trigger_log_dir(1, dead.id)}/rotated/trigger.log",
+ get_trigger_log_relative_path(1, dead.id),
+ get_trigger_log_relative_path(3, gone),
+ ]
+ )
+ assert uploads.index(f"{get_trigger_log_dir(1,
dead.id)}/rotated/trigger.log") < uploads.index(
+ get_trigger_log_relative_path(1, dead.id)
+ )
+
+ @pytest.mark.parametrize("remote", [True, False], ids=["remote", "local"])
+ def test_task_logs_of_a_dag_called_triggers_are_not_taken_for_orphans(
+ self, supervisor_builder, session, mocker, tmp_path, remote
+ ):
+ own = supervisor_builder()
+ dead = Job(
+ job_type="TriggererJob",
+ state="running",
+ latest_heartbeat=timezone.utcnow() - datetime.timedelta(hours=1),
+ )
+ session.add(dead)
+ session.flush()
+ # Dag "triggers", run "1234", try 2, with log_filename_template
+ # "{{ ti.dag_id }}/{{ ti.run_id }}/{{ try_number }}/{{ ti.task_id
}}.log".
+ task_log = tmp_path / "triggers/1234/2/watch.log"
+ orphan = tmp_path / get_trigger_log_relative_path(1, dead.id)
+ for path in (task_log, orphan):
+ path.parent.mkdir(parents=True)
+ path.write_text('{"n": 0}\n')
+ store = self.S3RemoteLogIO(tmp_path) if remote else None
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ if store is not None:
+ assert [path for path, _, _ in store.uploads] ==
[orphan.relative_to(tmp_path).as_posix()]
+ assert task_log.read_text() == '{"n": 0}\n'
+ assert not orphan.parent.exists()
+
+ def
test_orphans_are_looked_for_at_start_and_once_more_after_the_health_threshold(
+ self, jobless_supervisor, mocker
+ ):
+ clock = mocker.patch("airflow.utils.log.watcher_log.time.monotonic",
return_value=1000.0)
+ upload = mocker.patch.object(WatcherLogs,
"handle_orphaned_trigger_logs", autospec=True)
+
+ with conf_vars({("triggerer", "triggerer_health_check_threshold"):
"30"}):
+ for now in (1000.0, 1010.0, 1031.0, 1100.0):
+ clock.return_value = now
+
jobless_supervisor.watcher_logs.maybe_handle_orphaned_trigger_logs(None)
+ if jobless_supervisor.watcher_logs._orphan_scan_thread is not
None:
+ jobless_supervisor.watcher_logs._orphan_scan_thread.join()
+
+ assert upload.call_count == 2
+
+ @pytest.mark.parametrize("to_stdout", [True, False])
+ def test_a_full_disk_does_not_stop_the_triggerer(
+ self, jobless_supervisor, mocker, tmp_path, cap_structlog, to_stdout
+ ):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ jobless_supervisor.logger_cache[5] = WatcherLoggingFactory(
+ log_path=get_trigger_log_relative_path(5, 1),
+ trigger_id=5,
+ to_stdout=to_stdout,
+ on_error=jobless_supervisor._warn_trigger_log_error,
+ )
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ gen.send(msgspec.json.encode({"event": "poll 1", "level": "info",
"trigger_id": 5}))
+ underlying =
jobless_supervisor.logger_cache[5].bound_logger._logger
+ # The disk fills up: every write fails, also after the file is
reopened.
+ full = mocker.MagicMock(
+ closed=False, **{"write.side_effect": OSError(28, "No space
left on device")}
+ )
+ full.close.side_effect = lambda: setattr(full, "closed", True)
+ mocker.patch("pathlib.Path.open", return_value=full)
+ underlying._file, underlying._write = full, full.write
+ for n in (2, 3):
+ gen.send(msgspec.json.encode({"event": f"poll {n}", "level":
"info", "trigger_id": 5}))
+ jobless_supervisor.close_trigger_log_files()
+
+ events = [e["event"] for e in cap_structlog]
+ # Every line still reaches the triggerer's output, once, and the
failure is reported once.
+ assert [events.count(f"poll {n}") for n in (1, 2, 3)] ==
[int(to_stdout), 1, 1]
+ assert (
+ events.count(
+ "Could not write a trigger's log file; its lines go to the
triggerer's output meanwhile"
+ )
+ == 1
+ )
+ assert full.closed
+
+ def test_a_log_folder_that_cannot_be_created_does_not_stop_the_triggerer(
+ self, jobless_supervisor, mocker, tmp_path, cap_structlog
+ ):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ (tmp_path / "triggers").write_text("a file where the folder should be")
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ jobless_supervisor.logger_cache[5] = WatcherLoggingFactory(
+ log_path=get_trigger_log_relative_path(5, 1), trigger_id=5,
to_stdout=False
+ )
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ gen.send(msgspec.json.encode({"event": "poll 1", "level": "info",
"trigger_id": 5}))
+
+ assert {"event": "poll 1", "trigger_id": 5} in cap_structlog
+
+ def test_a_failed_flush_on_close_does_not_stop_the_triggerer(
+ self, jobless_supervisor, mocker, cap_structlog
+ ):
+ own_file = mocker.MagicMock(spec=WatcherLoggingFactory)
+ own_file.close.side_effect = OSError(28, "No space left on device")
+ jobless_supervisor.logger_cache[3] = own_file
+
+ jobless_supervisor.close_trigger_log_files()
+
+ assert {"trigger_id": 3, "error": "[Errno 28] No space left on
device"} in cap_structlog
+ assert jobless_supervisor.logger_cache == {}
+
+ @mock.patch("airflow.jobs.triggerer_job_runner.time.monotonic")
+ def test_a_log_file_error_is_reported_at_most_once_a_minute(
+ self, clock, jobless_supervisor, cap_structlog
+ ):
+ for now in (100.0, 130.0, 161.0):
+ clock.return_value = now
+ jobless_supervisor._warn_trigger_log_error(5, OSError(28, "No
space left on device"))
+
+ assert [e.get("trigger_id") for e in cap_structlog if e.get("error")]
== [5, 5]
+
+ @pytest.mark.parametrize(
+ "error",
+ [OSError(28, "No space left on device"), OSError(24, "Too many open
files")],
+ ids=["write", "open"],
+ )
+ def
test_a_task_trigger_log_that_cannot_be_written_does_not_stop_the_triggerer(
+ self, jobless_supervisor, mocker, tmp_path, cap_structlog, error
+ ):
+ mocker.patch("airflow.sdk.log.configure_logging")
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ factory = TriggerLoggingFactory(
+ log_path="dag_id=d/attempt=1.log.trigger.1.log",
+ ti=mocker.Mock(),
+ trigger_id=5,
+ on_error=jobless_supervisor._warn_trigger_log_error,
+ )
+ jobless_supervisor.logger_cache[5] = factory
+ gen = jobless_supervisor._process_log_messages_from_subprocess()
+ next(gen)
+ if error.errno == 24:
+ mocker.patch("pathlib.Path.open", side_effect=error)
+ else:
+ gen.send(msgspec.json.encode({"event": "poll 1", "level":
"info", "trigger_id": 5}))
+ mocker.patch.object(factory.bound_logger._logger, "_write",
side_effect=error)
+ gen.send(msgspec.json.encode({"event": "poll 2", "level": "info",
"trigger_id": 5}))
+ gen.send(msgspec.json.encode({"event": "poll 3", "level": "info",
"trigger_id": 5}))
+
+ assert {"event": "poll 2", "trigger_id": 5} in cap_structlog
+ assert {"event": "poll 3", "trigger_id": 5} in cap_structlog
+ assert {"trigger_id": 5, "error": str(error)} in cap_structlog
+
+ @pytest.mark.parametrize(
+ ("setting", "soft_limit", "expected"),
+ [
+ pytest.param(256, 4096, 256,
id="setting-below-a-quarter-of-the-limit"),
+ pytest.param(256, 256, 64, id="limit-as-low-as-the-setting"),
+ pytest.param(0, 512, 128, id="no-limit-set"),
+ ],
+ )
+ def test_open_files_stay_below_the_processs_limit(self, mocker, setting,
soft_limit, expected):
+ mocker.patch("resource.getrlimit", return_value=(soft_limit,
soft_limit))
+ with conf_vars({("triggerer", "watcher_log_max_open_files"):
str(setting)}):
+ assert get_watcher_log_max_open_files() == expected
+
+ def
test_without_remote_logging_only_the_newest_files_of_dead_triggerers_are_kept(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ alive = Job(job_type="TriggererJob", state="running",
latest_heartbeat=now)
+ older, newer = (
+ Job(job_type="TriggererJob", state="running", latest_heartbeat=now
- datetime.timedelta(hours=h))
+ for h in (2, 1)
+ )
+ one, two =
(Trigger(classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}) for _
in range(2))
+ session.add_all([alive, older, newer, one, two])
+ session.flush()
+ paths = [
+ get_trigger_log_relative_path(one.id, older.id),
+ f"{get_trigger_log_dir(one.id, older.id)}/rotated/trigger.log",
+ get_trigger_log_relative_path(one.id, newer.id),
+ get_trigger_log_relative_path(two.id, older.id),
+ get_trigger_log_relative_path(one.id, alive.id),
+ get_trigger_log_relative_path(one.id, own.job.id),
+ get_trigger_log_relative_path(one.id, uuid.uuid4()),
+ ]
+ for path in paths:
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("line\n")
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=None)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert [p for p in paths if (tmp_path / p).exists()] == paths[2:]
+
+ def test_without_remote_logging_the_most_recently_written_files_are_kept(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ # On a log folder the triggerers share, an older triggerer took the
watcher over when a newer one
+ # died, and wrote its newest lines.
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ older, newer = (
+ Job(job_type="TriggererJob", state="running", latest_heartbeat=now
- datetime.timedelta(hours=h))
+ for h in (2, 1)
+ )
+ trigger = Trigger(classpath="airflow.triggers.testing.SuccessTrigger",
kwargs={})
+ session.add_all([older, newer, trigger])
+ session.flush()
+ for job, written in ((older, time.time()), (newer, time.time() -
3600)):
+ path = tmp_path / get_trigger_log_relative_path(trigger.id, job.id)
+ path.parent.mkdir(parents=True)
+ path.write_text("line\n")
+ os.utime(path, (written, written))
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=None)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert [
+ p.relative_to(tmp_path).as_posix() for p in (tmp_path /
get_trigger_log_dir(trigger.id)).iterdir()
+ ] == [get_trigger_log_dir(trigger.id, older.id)]
+
+ def
test_without_remote_logging_nothing_is_kept_of_triggers_that_no_longer_exist(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ alive = Job(job_type="TriggererJob", state="running",
latest_heartbeat=now)
+ older, newer = (
+ Job(job_type="TriggererJob", state="running", latest_heartbeat=now
- datetime.timedelta(hours=h))
+ for h in (2, 1)
+ )
+ session.add_all([alive, older, newer])
+ session.flush()
+ # Ids no trigger row has, e.g. of a watcher whose kwargs changed.
+ gone, gone_with_live = (max(session.scalars(select(Trigger.id)),
default=0) + n for n in (1000, 1001))
+ paths = [
+ get_trigger_log_relative_path(gone, older.id),
+ get_trigger_log_relative_path(gone, newer.id),
+ f"{get_trigger_log_dir(gone, newer.id)}/rotated/trigger.log",
+ get_trigger_log_relative_path(gone_with_live, newer.id),
+ get_trigger_log_relative_path(gone_with_live, alive.id),
+ ]
+ for path in paths:
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("line\n")
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=None)
+
mocker.patch("airflow.utils.log.watcher_log._TRIGGER_ID_QUERY_CHUNK_SIZE", 1)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ # A live triggerer's file is never touched, even for a trigger that is
gone.
+ assert [p for p in paths if (tmp_path / p).exists()] == [paths[4]]
+ assert not (tmp_path / get_trigger_log_dir(gone)).exists()
+
+ def test_a_remote_handler_not_known_by_name_is_logged_once(self,
cap_structlog):
+ # Handlers are told apart by class name: a renamed one must leave a
trace, and no built-in one may.
+ built_in = [
+ type(name, (), {})()
+ for name in (
+ "S3RemoteLogIO",
+ "CloudWatchRemoteLogIO",
+ "GCSRemoteLogIO",
+ "CloudLoggingRemoteLogIO",
+ "WasbRemoteLogIO",
+ "OSSRemoteLogIO",
+ "HdfsRemoteLogIO",
+ "ElasticsearchRemoteLogIO",
+ "OpensearchRemoteLogIO",
+ )
+ ]
+ built_in.append(type("StackdriverRemoteLogIO", (type(built_in[3]),),
{})())
+ unknown = type("RenamedS3RemoteLogIO", (), {})()
+ for handler in (*built_in, *built_in, unknown, unknown):
+ _handler_keeps_no_copy(handler)
+ _handler_is(handler, _EMPTIES_STORED_FILE_HANDLERS)
+
+ logged = [
+ e["handler"] for e in cap_structlog if
e["event"].startswith("Remote log handler not known")
+ ]
+ assert logged == [f"{__name__}.RenamedS3RemoteLogIO"]
+
+ @pytest.mark.parametrize(
+ ("handler_name", "processors"),
+ [
+ pytest.param("ElasticsearchRemoteLogIO", (), id="task-logs-only"),
+ pytest.param(
+ "StreamingRemoteLogIO", (lambda logger, name, event: event,),
id="streams-each-line"
+ ),
+ ],
+ )
+ def
test_files_a_remote_handler_keeps_are_removed_as_without_remote_logging(
+ self, supervisor_builder, session, mocker, tmp_path, handler_name,
processors
+ ):
+ # Elasticsearch and OpenSearch take only task logs; CloudWatch has
each line already and keeps the file.
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ older, newer = (
+ Job(job_type="TriggererJob", state="running", latest_heartbeat=now
- datetime.timedelta(hours=h))
+ for h in (2, 1)
+ )
+ trigger = Trigger(classpath="airflow.triggers.testing.SuccessTrigger",
kwargs={})
+ session.add_all([older, newer, trigger])
+ session.flush()
+ gone = trigger.id + 1000
+ paths = [
+ get_trigger_log_relative_path(trigger.id, older.id),
+ get_trigger_log_relative_path(trigger.id, newer.id),
+ get_trigger_log_relative_path(gone, newer.id),
+ ]
+ for n, path in enumerate(paths):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("line\n")
+ os.utime(tmp_path / path, (time.time() - 3600 + n, time.time() -
3600 + n))
+ handler = type(handler_name, (mock.Mock,), {"processors":
processors})()
+ handler.upload.side_effect = lambda path, ti: (tmp_path /
path).read_text()
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=handler)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert sorted(c.args[0] for c in handler.upload.call_args_list) ==
sorted(paths)
+ assert [p for p in paths if (tmp_path / p).exists()] == [paths[1]]
+ assert not list(tmp_path.glob("triggers/*/*/.uploading"))
+
+ def test_files_a_failed_upload_left_are_kept_for_a_later_start(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ # S3, GCS and WASB log a failed upload and leave the file as it was:
it is not a handler without a copy.
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ older, newer = (
+ Job(job_type="TriggererJob", state="running", latest_heartbeat=now
- datetime.timedelta(hours=h))
+ for h in (2, 1)
+ )
+ trigger = Trigger(classpath="airflow.triggers.testing.SuccessTrigger",
kwargs={})
+ session.add_all([older, newer, trigger])
+ session.flush()
+ paths = [get_trigger_log_relative_path(trigger.id, job.id) for job in
(older, newer)]
+ for n, path in enumerate(paths):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("line\n")
+ os.utime(tmp_path / path, (time.time() - 3600 + n, time.time() -
3600 + n))
+ store = self.S3RemoteLogIO(tmp_path)
+ store.quiet_failures = len(paths)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert store.attempts == len(paths)
+ assert [p for p in paths if (tmp_path / p).exists()] == paths
+
+ @pytest.mark.parametrize("store_class", ["HdfsRemoteLogIO",
"OSSRemoteLogIO", "CustomRemoteLogIO"])
+ def test_files_a_store_keeps_after_uploading_them_are_uploaded_once(
+ self, supervisor_builder, session, mocker, tmp_path, store_class
+ ):
+ # HDFS and OSS keep the file they stored unless delete_local_logs is
on, and OSS appends.
+ dead = Job(
+ job_type="TriggererJob",
+ state="running",
+ latest_heartbeat=timezone.utcnow() - datetime.timedelta(hours=1),
+ )
+ session.add(dead)
+ session.flush()
+ paths = [
+ f"{get_trigger_log_dir(1, dead.id)}/rotated/trigger.000003.log",
+ get_trigger_log_relative_path(1, dead.id),
+ ]
+ for n, path in enumerate(paths):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text(json.dumps({"n": n}) + "\n")
+ store = getattr(self, store_class)(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ own = supervisor_builder()
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+ own.watcher_logs.handle_orphaned_trigger_logs(
+ own.job.id
+ ) # a triggerer scans at start and once more
+
+ assert sorted(path for path, _, _ in store.uploads) == sorted(paths)
+ assert store.lines() == list(range(len(paths)))
+ assert not (tmp_path / get_trigger_log_dir(1)).exists()
+
+ def test_uploaded_files_of_dead_triggerers_leave_no_directories(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ # Without delete_local_logs, S3, GCS and WASB empty the file they
uploaded and leave its directory.
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ alive = Job(job_type="TriggererJob", state="running",
latest_heartbeat=now)
+ dead = Job(
+ job_type="TriggererJob", state="running", latest_heartbeat=now -
datetime.timedelta(hours=1)
+ )
+ session.add_all([alive, dead])
+ session.flush()
+ for path in (
+ f"{get_trigger_log_dir(1, dead.id)}/rotated/trigger.000003.log",
+ get_trigger_log_relative_path(1, dead.id),
+ get_trigger_log_relative_path(2, dead.id),
+ get_trigger_log_relative_path(2, alive.id),
+ get_trigger_log_relative_path(3, own.job.id),
+ ):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text('{"n": 0}\n')
+ (tmp_path / get_trigger_log_dir(4, dead.id)).mkdir(parents=True) #
emptied by an earlier start
+ (tmp_path / get_trigger_log_relative_path(4, dead.id)).touch()
+ store = self._Store(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert len(store.lines()) == 3
+ assert sorted(p.relative_to(tmp_path).as_posix() for p in
tmp_path.glob("triggers/*/*")) == sorted(
+ [get_trigger_log_dir(2, alive.id), get_trigger_log_dir(3,
own.job.id)]
+ )
+ assert sorted(p.relative_to(tmp_path).as_posix() for p in
tmp_path.glob("triggers/*")) == [
+ get_trigger_log_dir(2),
+ get_trigger_log_dir(3),
+ ]
+
+ def test_files_another_triggerer_is_uploading_are_skipped(
+ self, supervisor_builder, session, mocker, tmp_path
+ ):
+ # Triggerers that start together on a shared log folder see the same
dead triggerer's files.
+ now = timezone.utcnow()
+ own = supervisor_builder()
+ dead = Job(
+ job_type="TriggererJob", state="running", latest_heartbeat=now -
datetime.timedelta(hours=1)
+ )
+ session.add(dead)
+ session.flush()
+ for trigger_id in (1, 2, 3):
+ (tmp_path / get_trigger_log_dir(trigger_id,
dead.id)).mkdir(parents=True)
+ (tmp_path / get_trigger_log_relative_path(trigger_id,
dead.id)).write_text("line\n")
+ # Trigger 2's files are being uploaded by another triggerer; trigger
3's claim was left by one that
+ # died while uploading.
+ (tmp_path / f"{get_trigger_log_dir(2, dead.id)}/.uploading").touch()
+ stale = tmp_path / f"{get_trigger_log_dir(3, dead.id)}/.uploading"
+ stale.touch()
+ os.utime(stale, (time.time() - 3600, time.time() - 3600))
+ store = self.S3RemoteLogIO(tmp_path)
+ mocker.patch("airflow.sdk.log.load_remote_log_handler",
return_value=store)
+
+ with conf_vars({("logging", "base_log_folder"): str(tmp_path)}):
+ own.watcher_logs.handle_orphaned_trigger_logs(own.job.id)
+
+ assert sorted(path for path, _, _ in store.uploads) == [
+ get_trigger_log_relative_path(1, dead.id),
+ get_trigger_log_relative_path(3, dead.id),
+ ]
+ assert [p.relative_to(tmp_path).as_posix() for p in
tmp_path.glob("triggers/*/*/.uploading")] == [
+ f"{get_trigger_log_dir(2, dead.id)}/.uploading"
+ ]
+
+ def test_orphans_are_uploaded_off_the_main_loop(self, jobless_supervisor,
mocker):
+ # Thousands of files left by a killed triggerer are thousands of
uploads, each a timeout while the
+ # store is unreachable: meanwhile the main loop must still heartbeat
and handle events.
+ ran_in = []
+ done = threading.Event()
+
+ def scan(own_job_id):
+ ran_in.append(threading.current_thread())
+ done.set()
+
+ mocker.patch.object(WatcherLogs, "handle_orphaned_trigger_logs",
side_effect=scan)
+
+
jobless_supervisor.watcher_logs.maybe_handle_orphaned_trigger_logs(None)
+
+ assert done.wait(5)
+ assert ran_in != [threading.current_thread()]
+
+ def test_released_watchers_files_are_uploaded_off_the_main_loop(self,
jobless_supervisor, mocker):
+ # A triggerer whose watchers were taken over (after losing the
database for a while) releases them
+ # all at once.
+ release = threading.Event()
+ factory = mocker.MagicMock(spec=WatcherLoggingFactory)
+ factory.upload_to_remote.side_effect = lambda: release.wait(5)
+ jobless_supervisor.logger_cache[42] = factory
+
+ jobless_supervisor._close_trigger_logger(42)
+
+ factory.close.assert_not_called() # still uploading
+ release.set()
+ jobless_supervisor.watcher_logs._upload_pool.shutdown(wait=True)
+ factory.close.assert_called_once()
+
+ def
test_a_watcher_back_while_its_file_uploads_starts_once_the_upload_is_done(
+ self, jobless_supervisor, mocker
+ ):
+ # The handler empties or removes the file it uploaded, so the new
instance must not write to it yet,
+ # but the main loop must not wait either (minutes per upload with an
unreachable store).
+ started, release = threading.Event(), threading.Event()
+ factory = mocker.MagicMock(spec=WatcherLoggingFactory)
+ factory.upload_to_remote.side_effect = lambda: (started.set(),
release.wait(5))
+ jobless_supervisor.logger_cache[42] = factory
+ jobless_supervisor._close_trigger_logger(42)
+ upload = jobless_supervisor.watcher_logs._pending_uploads[42]
+ assert started.wait(5) # in flight: one not started yet is cancelled
instead
+
+ self._create(jobless_supervisor, self._watcher_trigger(mocker, 42),
mocker)
+ assert 42 not in jobless_supervisor.logger_cache
+
+ release.set()
+ upload.result(timeout=5)
+ self._create(jobless_supervisor, self._watcher_trigger(mocker, 42),
mocker)
+ assert jobless_supervisor.logger_cache[42] is not factory
+
+ def
test_a_watcher_back_before_its_upload_starts_runs_at_once_on_the_same_file(
+ self, jobless_supervisor, mocker
+ ):
+ # Queued behind other uploads (minutes each with an unreachable
store): cancel it, keep the file.
+ release = threading.Event()
+ jobless_supervisor.watcher_logs._upload_pool =
ThreadPoolExecutor(max_workers=1)
+ busy =
jobless_supervisor.watcher_logs._upload_pool.submit(release.wait, 5)
+ factory = mocker.MagicMock(spec=WatcherLoggingFactory)
+ jobless_supervisor.logger_cache[42] = factory
+ jobless_supervisor._close_trigger_logger(42)
+ upload = jobless_supervisor.watcher_logs._pending_uploads[42]
+
+ self._create(jobless_supervisor, self._watcher_trigger(mocker, 42),
mocker)
+
+ assert upload.cancelled()
+ assert jobless_supervisor.logger_cache[42] is factory
+ assert 42 not in jobless_supervisor.watcher_logs._pending_uploads
+ assert 42 not in jobless_supervisor.watcher_logs._released_factories
+ release.set()
+ busy.result(timeout=5)
+ jobless_supervisor.watcher_logs._upload_pool.shutdown(wait=True)
+ factory.upload_to_remote.assert_not_called()
+ factory.close.assert_not_called()
+
+ def test_open_files_are_uploaded_side_by_side_at_stop(self,
jobless_supervisor, mocker):
+ # One after the other, thousands of watchers' files take longer than a
container's stop timeout.
+ both = threading.Barrier(2, timeout=5)
+ factories = {t: mocker.MagicMock(spec=WatcherLoggingFactory) for t in
(1, 2)}
+ for factory in factories.values():
+ factory.upload_to_remote.side_effect = both.wait
+ jobless_supervisor.logger_cache.update(factories)
+
+ jobless_supervisor.close_trigger_log_files()
+
+ assert not both.broken
+ for factory in factories.values():
+ factory.close.assert_called_once()
+
+ def test_defaults(self):
Review Comment:
No need to have a test for that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]