This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7cfa1be467 Completed D400 for multiple folders (#27722)
7cfa1be467 is described below
commit 7cfa1be467b995b886a97b68498137a76a31f97c
Author: Dov Benyomin Sohacheski <[email protected]>
AuthorDate: Wed Nov 16 22:48:51 2022 +0200
Completed D400 for multiple folders (#27722)
---
airflow/datasets/manager.py | 3 +++
airflow/decorators/base.py | 5 +++--
airflow/decorators/branch_python.py | 3 ++-
airflow/exceptions.py | 10 ++++++----
airflow/hooks/base.py | 8 ++++++--
airflow/hooks/subprocess.py | 2 +-
airflow/macros/__init__.py | 10 +++++-----
airflow/security/kerberos.py | 9 ++++++---
airflow/timetables/events.py | 5 +++--
airflow/triggers/base.py | 4 +++-
airflow/triggers/temporal.py | 4 ++++
11 files changed, 42 insertions(+), 21 deletions(-)
diff --git a/airflow/datasets/manager.py b/airflow/datasets/manager.py
index 937bd04fb8..5f002b4006 100644
--- a/airflow/datasets/manager.py
+++ b/airflow/datasets/manager.py
@@ -46,6 +46,8 @@ class DatasetManager(LoggingMixin):
self, *, task_instance: TaskInstance, dataset: Dataset, extra=None,
session: Session, **kwargs
) -> None:
"""
+ Register dataset related changes.
+
For local datasets, look them up, record the dataset event, queue
dagruns, and broadcast
the dataset event
"""
@@ -107,6 +109,7 @@ class DatasetManager(LoggingMixin):
def resolve_dataset_manager() -> DatasetManager:
+ """Retrieve the dataset manager."""
_dataset_manager_class = conf.getimport(
section="core",
key="dataset_manager_class",
diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py
index b56ea42149..441c509591 100644
--- a/airflow/decorators/base.py
+++ b/airflow/decorators/base.py
@@ -113,8 +113,9 @@ def get_unique_task_id(
task_group: TaskGroup | None = None,
) -> str:
"""
- Generate unique task id given a DAG (or if run in a DAG context)
- Ids are generated by appending a unique number to the end of
+ Generate unique task id given a DAG (or if run in a DAG context).
+
+ IDs are generated by appending a unique number to the end of
the original task id.
Example:
diff --git a/airflow/decorators/branch_python.py
b/airflow/decorators/branch_python.py
index b27d27b93d..b7e3a94826 100644
--- a/airflow/decorators/branch_python.py
+++ b/airflow/decorators/branch_python.py
@@ -70,7 +70,8 @@ def branch_task(
python_callable: Callable | None = None, multiple_outputs: bool | None =
None, **kwargs
) -> TaskDecorator:
"""
- Wraps a python function into a BranchPythonOperator
+ Wraps a python function into a BranchPythonOperator.
+
For more information on how to use this operator, take a look at the guide:
:ref:`howto/operator:BranchPythonOperator`
diff --git a/airflow/exceptions.py b/airflow/exceptions.py
index e3c4e333b9..a60e3f90ff 100644
--- a/airflow/exceptions.py
+++ b/airflow/exceptions.py
@@ -17,7 +17,7 @@
# under the License.
# Note: Any AirflowException raised is expected to cause the TaskInstance
# to be marked in an ERROR state
-"""Exceptions used by Airflow"""
+"""Exceptions used by Airflow."""
from __future__ import annotations
import datetime
@@ -84,7 +84,7 @@ class AirflowWebServerTimeout(AirflowException):
class AirflowSkipException(AirflowException):
- """Raise when the task should be skipped"""
+ """Raise when the task should be skipped."""
class AirflowFailException(AirflowException):
@@ -229,7 +229,7 @@ class SerializationError(AirflowException):
class ParamValidationError(AirflowException):
- """Raise when DAG params is invalid"""
+ """Raise when DAG params is invalid."""
class TaskNotFound(AirflowNotFoundException):
@@ -317,6 +317,8 @@ class ConnectionNotUnique(AirflowException):
class TaskDeferred(BaseException):
"""
+ Signal an operator moving to deferred state.
+
Special exception raised to signal that the operator it was raised from
wishes to defer until a trigger fires.
"""
@@ -347,7 +349,7 @@ class TaskDeferralError(AirflowException):
class PodMutationHookException(AirflowException):
- """Raised when exception happens during Pod Mutation Hook execution"""
+ """Raised when exception happens during Pod Mutation Hook execution."""
class PodReconciliationError(AirflowException):
diff --git a/airflow/hooks/base.py b/airflow/hooks/base.py
index a0c9f3d8f6..85467e7312 100644
--- a/airflow/hooks/base.py
+++ b/airflow/hooks/base.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Base class for all hooks"""
+"""Base class for all hooks."""
from __future__ import annotations
import logging
@@ -34,7 +34,9 @@ log = logging.getLogger(__name__)
class BaseHook(LoggingMixin):
"""
- Abstract base class for hooks, hooks are meant as an interface to
+ Abstract base class for hooks.
+
+ Hooks are meant as an interface to
interact with external systems. MySqlHook, HiveHook, PigHook return
object that can handle the connection and interaction to specific
instances of these systems, and expose consistent methods to interact
@@ -160,6 +162,8 @@ class DiscoverableHook(Protocol):
@staticmethod
def get_ui_field_behaviour() -> dict[str, Any]:
"""
+ Attributes of the UI field.
+
Returns dictionary describing customizations to implement in
javascript handling the
connection form. Should be compliant with
airflow/customized_form_field_behaviours.schema.json'
diff --git a/airflow/hooks/subprocess.py b/airflow/hooks/subprocess.py
index 41abf36457..b5a30b06ae 100644
--- a/airflow/hooks/subprocess.py
+++ b/airflow/hooks/subprocess.py
@@ -29,7 +29,7 @@ SubprocessResult = namedtuple("SubprocessResult",
["exit_code", "output"])
class SubprocessHook(BaseHook):
- """Hook for running processes with the ``subprocess`` module"""
+ """Hook for running processes with the ``subprocess`` module."""
def __init__(self) -> None:
self.sub_process: Popen[bytes] | None = None
diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py
index d5730c20b0..4364d3278a 100644
--- a/airflow/macros/__init__.py
+++ b/airflow/macros/__init__.py
@@ -31,7 +31,7 @@ from airflow.macros import hive # noqa
def ds_add(ds: str, days: int) -> str:
"""
- Add or subtract days from a YYYY-MM-DD
+ Add or subtract days from a YYYY-MM-DD.
:param ds: anchor date in ``YYYY-MM-DD`` format to add to
:param days: number of days to add to the ds, you can use negative values
@@ -49,8 +49,7 @@ def ds_add(ds: str, days: int) -> str:
def ds_format(ds: str, input_format: str, output_format: str) -> str:
"""
- Takes an input string and outputs another string
- as specified in the output format
+ Output datetime string in a given format.
:param ds: input string which contains a date
:param input_format: input string format. E.g. %Y-%m-%d
@@ -66,8 +65,9 @@ def ds_format(ds: str, input_format: str, output_format: str)
-> str:
def datetime_diff_for_humans(dt: Any, since: DateTime | None = None) -> str:
"""
- Return a human-readable/approximate difference between two datetimes, or
- one and now.
+ Return a human-readable/approximate difference between datetimes.
+
+ When only one datetime is provided, the comparison will be based on now.
:param dt: The datetime to display the diff for
:param since: When to display the date from. If ``None`` then the diff is
diff --git a/airflow/security/kerberos.py b/airflow/security/kerberos.py
index 0fb71177a3..bc36f5e98c 100644
--- a/airflow/security/kerberos.py
+++ b/airflow/security/kerberos.py
@@ -32,7 +32,7 @@ from __future__ import annotations
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Kerberos security provider"""
+"""Kerberos security provider."""
import logging
import shlex
import subprocess
@@ -49,7 +49,7 @@ log = logging.getLogger(__name__)
def renew_from_kt(principal: str | None, keytab: str, exit_on_fail: bool =
True):
"""
- Renew kerberos token from keytab
+ Renew kerberos token from keytab.
:param principal: principal
:param keytab: keytab file
@@ -159,7 +159,10 @@ def perform_krb181_workaround(principal: str):
def detect_conf_var() -> bool:
- """Return true if the ticket cache contains "conf" information as is found
+ """
+ Autodetect the Kerberos ticket configuration.
+
+ Return true if the ticket cache contains "conf" information as is found
in ticket caches of Kerberos 1.8.1 or later. This is incompatible with the
Sun Java Krb5LoginModule in Java6, so we need to take an action to work
around it.
diff --git a/airflow/timetables/events.py b/airflow/timetables/events.py
index 546d61e7a3..a59f4fc5b2 100644
--- a/airflow/timetables/events.py
+++ b/airflow/timetables/events.py
@@ -27,8 +27,9 @@ from airflow.timetables.base import DagRunInfo, DataInterval,
TimeRestriction, T
class EventsTimetable(Timetable):
"""
- Timetable that schedules DAG runs at specific listed datetimes. Suitable
for
- predictable but truly irregular scheduling such as sporting events.
+ Timetable that schedules DAG runs at specific listed datetimes.
+
+ Suitable for predictable but truly irregular scheduling such as sporting
events.
:param event_dates: List of datetimes for the DAG to run at. Duplicates
will be ignored. Must be finite
and of reasonable size as it will be loaded in its
entirety.
diff --git a/airflow/triggers/base.py b/airflow/triggers/base.py
index 625a58a891..6bfc0883ef 100644
--- a/airflow/triggers/base.py
+++ b/airflow/triggers/base.py
@@ -70,7 +70,9 @@ class BaseTrigger(abc.ABC, LoggingMixin):
def cleanup(self) -> None:
"""
- Called when the trigger is no longer needed and it's being removed
+ Cleanup the trigger.
+
+ Called when the trigger is no longer needed, and it's being removed
from the active triggerer process.
"""
diff --git a/airflow/triggers/temporal.py b/airflow/triggers/temporal.py
index 7d75f321a6..3967940a7e 100644
--- a/airflow/triggers/temporal.py
+++ b/airflow/triggers/temporal.py
@@ -26,6 +26,8 @@ from airflow.utils import timezone
class DateTimeTrigger(BaseTrigger):
"""
+ Trigger based on a datetime.
+
A trigger that fires exactly once, at the given datetime, give or take
a few seconds.
@@ -68,6 +70,8 @@ class DateTimeTrigger(BaseTrigger):
class TimeDeltaTrigger(DateTimeTrigger):
"""
+ Create DateTimeTriggers based on delays.
+
Subclass to create DateTimeTriggers based on time delays rather
than exact moments.
