This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ea420eee2a D205 Support - Dag Processing (#32449)
ea420eee2a is described below
commit ea420eee2a6e37977ba16e1a6bdaeb92c0f86937
Author: D. Ferruzzi <[email protected]>
AuthorDate: Sat Jul 8 23:01:34 2023 -0700
D205 Support - Dag Processing (#32449)
---
airflow/dag_processing/manager.py | 25 +++++++++++++++----------
airflow/dag_processing/processor.py | 3 +++
2 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/airflow/dag_processing/manager.py
b/airflow/dag_processing/manager.py
index da024161b8..7e44e89816 100644
--- a/airflow/dag_processing/manager.py
+++ b/airflow/dag_processing/manager.py
@@ -167,11 +167,10 @@ class DagFileProcessorAgent(LoggingMixin,
MultiprocessingStartMethodMixin):
def run_single_parsing_loop(self) -> None:
"""
Should only be used when launched DAG file processor manager in sync
mode.
- Send agent heartbeat signal to the manager, requesting that it runs one
- processing "loop".
- Call wait_until_finished to ensure that any launched processors have
- finished before continuing
+ Send agent heartbeat signal to the manager, requesting that it runs
one processing "loop".
+
+ Call wait_until_finished to ensure that any launched processors have
finished before continuing.
"""
if not self._parent_signal_conn or not self._process:
raise ValueError("Process not started.")
@@ -477,6 +476,7 @@ class DagFileProcessorManager(LoggingMixin):
def start(self):
"""
Use multiple processes to parse and generate tasks for the DAGs in
parallel.
+
By processing them in separate processes, we can get parallelism and
isolation
from potentially harmful user code.
"""
@@ -519,6 +519,7 @@ class DagFileProcessorManager(LoggingMixin):
):
"""
Detects DAGs which are no longer present in files.
+
Deactivate them and remove them in the serialized_dag table.
"""
to_deactivate = set()
@@ -888,6 +889,7 @@ class DagFileProcessorManager(LoggingMixin):
def get_pid(self, file_path) -> int | None:
"""
Retrieve the PID of the process processing the given file or None if
the file is not being processed.
+
:param file_path: the path to the file that's being processed.
"""
if file_path in self._processors:
@@ -905,6 +907,7 @@ class DagFileProcessorManager(LoggingMixin):
def get_last_runtime(self, file_path) -> float | None:
"""
Retrieve the last processing time of a specific path.
+
:param file_path: the path to the file that was processed
:return: the runtime (in seconds) of the process of the last run, or
None if the file was never processed.
@@ -915,9 +918,9 @@ class DagFileProcessorManager(LoggingMixin):
def get_last_dag_count(self, file_path) -> int | None:
"""
Retrieve the total DAG count at a specific path.
+
:param file_path: the path to the file that was processed
- :return: the number of dags loaded from that file, or None if the file
- was never processed.
+ :return: the number of dags loaded from that file, or None if the file
was never processed.
"""
stat = self._file_stats.get(file_path)
return stat.num_dags if stat else None
@@ -925,9 +928,9 @@ class DagFileProcessorManager(LoggingMixin):
def get_last_error_count(self, file_path) -> int | None:
"""
Retrieve the total number of errors from processing a specific path.
+
:param file_path: the path to the file that was processed
- :return: the number of import errors from processing, or None if the
file
- was never processed.
+ :return: the number of import errors from processing, or None if the
file was never processed.
"""
stat = self._file_stats.get(file_path)
return stat.import_errors if stat else None
@@ -935,9 +938,9 @@ class DagFileProcessorManager(LoggingMixin):
def get_last_finish_time(self, file_path) -> datetime | None:
"""
Retrieve the last completion time for processing a specific path.
+
:param file_path: the path to the file that was processed
- :return: the finish time of the process of the last run, or None if the
- file was never processed.
+ :return: the finish time of the process of the last run, or None if
the file was never processed.
"""
stat = self._file_stats.get(file_path)
return stat.last_finish_time if stat else None
@@ -945,6 +948,7 @@ class DagFileProcessorManager(LoggingMixin):
def get_start_time(self, file_path) -> datetime | None:
"""
Retrieve the last start time for processing a specific path.
+
:param file_path: the path to the file that's being processed
:return: the start time of the process that's processing the
specified file or None if the file is not currently being
processed.
@@ -956,6 +960,7 @@ class DagFileProcessorManager(LoggingMixin):
def get_run_count(self, file_path) -> int:
"""
The number of times the given file has been parsed.
+
:param file_path: the path to the file that's being processed.
"""
stat = self._file_stats.get(file_path)
diff --git a/airflow/dag_processing/processor.py
b/airflow/dag_processing/processor.py
index 9ee865bbf8..5c175571c0 100644
--- a/airflow/dag_processing/processor.py
+++ b/airflow/dag_processing/processor.py
@@ -598,6 +598,7 @@ class DagFileProcessor(LoggingMixin):
) -> None:
"""
Update any import errors to be displayed in the UI.
+
For the DAGs in the given DagBag, record any associated import errors
and clears
errors for files that no longer have them. These are usually displayed
through the
Airflow UI so that users know that there are issues parsing DAGs.
@@ -664,6 +665,7 @@ class DagFileProcessor(LoggingMixin):
def update_dag_warnings(self, *, session: Session, dagbag: DagBag) -> None:
"""
Update any import warnings to be displayed in the UI.
+
For the DAGs in the given DagBag, record any associated configuration
warnings and clear
warnings for files that no longer have them. These are usually
displayed through the
Airflow UI so that users know that there are issues parsing DAGs.
@@ -691,6 +693,7 @@ class DagFileProcessor(LoggingMixin):
) -> None:
"""
Execute on failure callbacks.
+
These objects can come from SchedulerJobRunner or from
DagProcessorJobRunner.
:param dagbag: Dag Bag of dags