Lee-W commented on code in PR #37101:
URL: https://github.com/apache/airflow/pull/37101#discussion_r1498745854
##########
airflow/models/dataset.py:
##########
@@ -341,9 +341,9 @@ def __repr__(self) -> str:
class DatasetBooleanCondition:
"""
- Base class for boolean logic for dataset triggers.
+ Base class for boolean conditions on datasets. This class is intended for
internal use only.
- :meta private:
+ :param objects: A variable number of Dataset, DatasetAny, or DatasetAll
instances.
Review Comment:
If we already know the object types to expect, should we annotate the one in
`__init__`
##########
airflow/models/dataset.py:
##########
@@ -341,9 +341,9 @@ def __repr__(self) -> str:
class DatasetBooleanCondition:
"""
- Base class for boolean logic for dataset triggers.
+ Base class for boolean conditions on datasets. This class is intended for
internal use only.
- :meta private:
Review Comment:
If it's for internal use only, why do we remove `:meta private`
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
Review Comment:
```suggestion
def __init__(self, *objects: Dataset | DatasetAny | DatasetAll) -> None:
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
+ self.value = value # value can be 'Dataset', '&', or '|'
+ self.left = left
+ self.right = right
+
+ def __or__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("|", self, other)
+
+ def __and__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("&", self, other)
+
+ def __repr__(self):
Review Comment:
```suggestion
def __repr__(self) -> str:
```
##########
tests/models/test_datasets.py:
##########
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+
+from airflow.models.dataset import Dataset, DatasetAll, DatasetAny,
extract_datasets
+
+
+def datasets_equal(d1, d2):
Review Comment:
Should we implement `__eq__` in `Dataset`, `DatasetAny` `DatasetAll` instead?
##########
docs/apache-airflow/authoring-and-scheduling/datasets.rst:
##########
@@ -237,6 +237,85 @@ Example:
Note that this example is using `(.values() | first | first)
<https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.first>`_
to fetch the first of one Dataset given to the DAG, and the first of one
DatasetEvent for that Dataset. An implementation may be quite complex if you
have multiple Datasets, potentially with multiple DatasetEvents.
+Advanced Dataset Scheduling with Conditional Expressions
+--------------------------------------------------------
+
+Apache Airflow introduces advanced scheduling capabilities that leverage
conditional expressions with datasets. This feature allows Airflow users to
define complex dependencies for DAG executions based on dataset updates, using
logical operators for more granular control over workflow triggers.
+
+Logical Operators for Datasets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Airflow supports two logical operators for combining dataset conditions:
+
+- **AND (``&``)**: Specifies that the DAG should be triggered only after all
of the specified datasets have been updated.
+- **OR (``|``)**: Specifies that the DAG should be triggered when any one of
the specified datasets is updated.
+
+These operators enable the expression of complex dataset update conditions,
enhancing the dynamism and flexibility of Airflow workflows.
+
+Example Usage
+-------------
+
+**Scheduling Based on Multiple Dataset Updates**
+
+To schedule a DAG to run only when two specific datasets have both been
updated, use the AND operator (``&``):
+
+.. code-block:: python
Review Comment:
Should we use `exampleinclude` instead?
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
+ self.value = value # value can be 'Dataset', '&', or '|'
+ self.left = left
+ self.right = right
+
+ def __or__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("|", self, other)
+
+ def __and__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("&", self, other)
+
+ def __repr__(self):
+ if isinstance(self.value, Dataset):
+ return f"Dataset(uri='{self.value.uri}')"
+ elif self.value == "&":
+ return repr(DatasetAll(self.left, self.right))
+ elif self.value == "|":
+ return repr(DatasetAny(self.left, self.right))
+
Review Comment:
Do we need to handle `else`?
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
Review Comment:
```suggestion
return f"DatasetAll({', '.join(map(str, self.objects))})"
```
I guess it should be `DatasetAll` instead?
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
Review Comment:
```suggestion
def __or__(self, other) -> DatasetAny:
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
Review Comment:
```suggestion
def __and__(self, other) -> DatasetAll:
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
Review Comment:
```suggestion
def __or__(self, other) -> DatasetAny:
```
##########
airflow/datasets/__init__.py:
##########
@@ -55,3 +55,13 @@ def __eq__(self, other):
def __hash__(self):
return hash(self.uri)
+
+ def __or__(self, other):
+ from airflow.models.dataset import DatasetAny
Review Comment:
May I know why do we want to import here? Is there anything expensive in
`airflow.models.dataset`
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
Review Comment:
```suggestion
def __and__(self, other) -> DatasetAll:
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
+ self.value = value # value can be 'Dataset', '&', or '|'
+ self.left = left
+ self.right = right
+
+ def __or__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("|", self, other)
+
+ def __and__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("&", self, other)
+
+ def __repr__(self):
+ if isinstance(self.value, Dataset):
+ return f"Dataset(uri='{self.value.uri}')"
+ elif self.value == "&":
+ return repr(DatasetAll(self.left, self.right))
+ elif self.value == "|":
+ return repr(DatasetAny(self.left, self.right))
+
+
+def extract_datasets(dataset_expression: DatasetsExpression | Dataset):
+ """
+ Extracts the dataset(s) from an DatasetsExpression.
+
+ :param dataset_expression: The DatasetsExpression to extract from.
+ """
+ if isinstance(dataset_expression, DatasetsExpression):
+ if dataset_expression.value == "&":
+ return DatasetAll(dataset_expression.left,
dataset_expression.right)
+ elif dataset_expression.value == "|":
+ return DatasetAny(dataset_expression.left,
dataset_expression.right)
+ else:
+ raise ValueError("Invalid Expression node value")
Review Comment:
```suggestion
raise ValueError("Invalid Expression node value")
```
##########
airflow/serialization/serialized_objects.py:
##########
@@ -407,6 +407,8 @@ def serialize_to_json(
serialized_object[key] = encode_timetable(value)
elif key == "dataset_triggers":
serialized_object[key] = cls.serialize(value)
+ elif key == "dataset_triggers":
+ serialized_object[key] = cls.serialize(value)
Review Comment:
```suggestion
elif key == "dataset_triggers":
serialized_object[key] = cls.serialize(value)
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
+ self.value = value # value can be 'Dataset', '&', or '|'
+ self.left = left
+ self.right = right
+
+ def __or__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("|", self, other)
+
+ def __and__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("&", self, other)
+
+ def __repr__(self):
+ if isinstance(self.value, Dataset):
+ return f"Dataset(uri='{self.value.uri}')"
+ elif self.value == "&":
+ return repr(DatasetAll(self.left, self.right))
+ elif self.value == "|":
+ return repr(DatasetAny(self.left, self.right))
+
+
+def extract_datasets(dataset_expression: DatasetsExpression | Dataset):
Review Comment:
```suggestion
def extract_datasets(dataset_expression: DatasetsExpression | Dataset) ->
DatasetBooleanCondition | DatasetsExpression:
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
+ self.value = value # value can be 'Dataset', '&', or '|'
+ self.left = left
+ self.right = right
+
+ def __or__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("|", self, other)
+
+ def __and__(self, other: Dataset | DatasetsExpression) ->
DatasetsExpression:
+ return DatasetsExpression("&", self, other)
+
+ def __repr__(self):
+ if isinstance(self.value, Dataset):
+ return f"Dataset(uri='{self.value.uri}')"
+ elif self.value == "&":
+ return repr(DatasetAll(self.left, self.right))
+ elif self.value == "|":
+ return repr(DatasetAny(self.left, self.right))
+
+
+def extract_datasets(dataset_expression: DatasetsExpression | Dataset):
+ """
+ Extracts the dataset(s) from an DatasetsExpression.
+
+ :param dataset_expression: The DatasetsExpression to extract from.
+ """
+ if isinstance(dataset_expression, DatasetsExpression):
+ if dataset_expression.value == "&":
+ return DatasetAll(dataset_expression.left,
dataset_expression.right)
+ elif dataset_expression.value == "|":
+ return DatasetAny(dataset_expression.left,
dataset_expression.right)
+ else:
+ raise ValueError("Invalid Expression node value")
+ else:
+ return dataset_expression
Review Comment:
```suggestion
return dataset_expression
```
##########
airflow/models/dataset.py:
##########
@@ -374,12 +386,96 @@ def all_datasets(self) -> dict[str, Dataset]:
class DatasetAny(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "and" relationship."""
+ """
+ Represents a logical OR condition of datasets.
+
+ Inherits from DatasetBooleanCondition.
+ """
agg_func = any
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(*self.objects, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(self, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAny({', '.join(map(str, self.objects))})"
+
class DatasetAll(DatasetBooleanCondition):
- """Use to combine datasets schedule references in an "or" relationship."""
+ """Represents a logical AND condition of datasets. Inherits from
DatasetBooleanCondition."""
agg_func = all
+
+ def __init__(self, *objects: Dataset | DatasetAny | DatasetAll):
+ """Initialize with one or more Dataset, DatasetAny, or DatasetAll
instances."""
+ super().__init__(*objects)
+
+ def __or__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAny(self, other)
+ return NotImplemented
+
+ def __and__(self, other):
+ if isinstance(other, (Dataset, DatasetAny, DatasetAll)):
+ return DatasetAll(*self.objects, other)
+ return NotImplemented
+
+ def __repr__(self) -> str:
+ return f"DatasetAnd({', '.join(map(str, self.objects))})"
+
+
+class DatasetsExpression:
+ """
+ Represents a node in an expression tree for dataset conditions.
+
+ :param value: The value of the node, which can be a 'Dataset', '&', or '|'.
+ :param left: The left child node.
+ :param right: The right child node.
+ """
+
+ def __init__(self, value, left=None, right=None):
Review Comment:
```suggestion
def __init__(self, value, left=None, right=None) -> None:
```
##########
tests/serialization/test_dag_serialization.py:
##########
@@ -554,6 +554,9 @@ def validate_deserialized_dag(self, serialized_dag, dag):
compare_serialization_list = {
"dataset_triggers",
}
+ compare_serialization_list = {
+ "dataset_triggers",
+ }
Review Comment:
```suggestion
compare_serialization_list = {
"dataset_triggers",
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]