This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4919400862c Add cross dependencies between airflow-core and task-sdk
(#49221)
4919400862c is described below
commit 4919400862c3fdaca9cde3f9d967ae7b4a662ba6
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Apr 14 18:26:55 2025 +0200
Add cross dependencies between airflow-core and task-sdk (#49221)
We have not managed fully unentangle dependencies between airlfow-core
and task-sdk and for now they are cross-dependent - both airflow-core
uses task-sdk and task-sdk uses airflow core. We are aiming to fix
this for Airflow 3.1.
We are planning to relax dependencies between airflow-core and
task-sdk in Airflow 3.1, but for now we are semi-closely tying the versions
because we are directly using imports between those packages and we
have no backwards-compatibility checks implemented to guard if there
are incompatible changes introduced.
The mechanism to prepare RC packages has been updated to make sure
that rcN version of apache-airflow, apache-airflow-core,
apache-airflow-task-sdk always refer to the same rc versions of each
other - which means that all three packages should be released always
together until we loosen relation between the packages.
We also update versions of just released providers that are Airflow 3
only.
---
.github/workflows/generate-constraints.yml | 5 ++++
airflow-core/pyproject.toml | 1 +
.../commands/release_management_commands.py | 13 ++++++++-
dev/breeze/src/airflow_breeze/global_constants.py | 10 +------
pyproject.toml | 14 ++++-----
.../ci/pre_commit/update_airflow_pyproject_toml.py | 34 +++++++++++++---------
scripts/in_container/run_generate_constraints.py | 18 ++++++++++--
task-sdk/pyproject.toml | 3 +-
task-sdk/src/airflow/sdk/__init__.py | 2 +-
9 files changed, 65 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 3aada42454d..0f740816863 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -115,6 +115,11 @@ jobs:
run: >
breeze release-management prepare-airflow-distributions
--distribution-format wheel --version-suffix-for-pypi dev0
+ - name: "Prepare task-sdk distribution"
+ shell: bash
+ run: >
+ breeze release-management prepare-task-sdk-distributions
+ --distribution-format wheel --version-suffix-for-pypi dev0
- name: "PyPI constraints"
shell: bash
timeout-minutes: 25
diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml
index c82addc136c..68271be4b35 100644
--- a/airflow-core/pyproject.toml
+++ b/airflow-core/pyproject.toml
@@ -136,6 +136,7 @@ dependencies = [
# Does not work with it Tracked in
https://github.com/fsspec/universal_pathlib/issues/276
"universal-pathlib>=0.2.2,!=0.2.4",
"uuid6>=2024.7.10",
+ "apache-airflow-task-sdk<1.1.0,>=1.0.0",
# pre-installed providers
"apache-airflow-providers-common-compat>=1.6.0",
"apache-airflow-providers-common-io>=1.5.3",
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 11bb1d0a33b..5277d99cc43 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -416,7 +416,7 @@ def update_version_suffix_in_pyproject_toml(version_suffix:
str, pyproject_toml_
f"[info]Not updating version suffix to {version_suffix}
for {line} as it already has the "
f"{version_suffix} suffix."
)
- if line.strip().startswith('"apache-airflow-providers-') and ">=" in
line:
+ if line.strip().startswith('"apache-airflow-') and ">=" in line:
if not line.endswith(
f'.{version_suffix}"',
):
@@ -438,6 +438,17 @@ def
update_version_suffix_in_pyproject_toml(version_suffix: str, pyproject_toml_
f"[info]Not updating version suffix to {version_suffix}
for {line} as it already has the "
f"{version_suffix} suffix."
)
+ if line.strip().startswith('"apache-airflow-task-sdk') and "==" in
line:
+ if not line.endswith(
+ f'.{version_suffix}",',
+ ):
+ get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
+ line = line.rstrip('",') + f'.{version_suffix}",'
+ else:
+ get_console().print(
+ f"[info]Not updating version suffix to {version_suffix}
for {line} as it already has the "
+ f"{version_suffix} suffix."
+ )
updated_lines.append(line)
new_content = "\n".join(updated_lines) + "\n"
get_console().print(f"[info]Writing updated content to
{pyproject_toml_path}.\n")
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 0e0c4057fde..b5655bf89ae 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -634,15 +634,7 @@ DEFAULT_EXTRAS = [
# END OF EXTRAS LIST UPDATED BY PRE COMMIT
]
-CHICKEN_EGG_PROVIDERS = " ".join(
- [
- "amazon",
- "common.messaging",
- "fab",
- "git",
- "openlineage",
- ]
-)
+CHICKEN_EGG_PROVIDERS = " ".join(["git", "common.messaging", "fab"])
PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [
diff --git a/pyproject.toml b/pyproject.toml
index 3b6502cc22a..a5aaefb7689 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -62,7 +62,7 @@ classifiers = [
version = "3.0.0"
dependencies = [
- "apache-airflow-task-sdk",
+ "apache-airflow-task-sdk<1.1.0,>=1.0.0",
"apache-airflow-core==3.0.0",
]
@@ -209,7 +209,7 @@ packages = []
"apache-airflow-providers-common-io>=1.4.2"
]
"common.messaging" = [
- "apache-airflow-providers-common-messaging"
+ "apache-airflow-providers-common-messaging>=1.0.0"
]
"common.sql" = [
"apache-airflow-providers-common-sql>=1.18.0"
@@ -251,7 +251,7 @@ packages = []
"apache-airflow-providers-ftp>=3.12.0"
]
"git" = [
- "apache-airflow-providers-git"
+ "apache-airflow-providers-git>=0.0.1"
]
"github" = [
"apache-airflow-providers-github>=2.8.0"
@@ -311,7 +311,7 @@ packages = []
"apache-airflow-providers-openfaas>=3.7.0"
]
"openlineage" = [
- "apache-airflow-providers-openlineage>=2.1.2"
+ "apache-airflow-providers-openlineage>=2.1.3"
]
"opensearch" = [
"apache-airflow-providers-opensearch>=1.5.0"
@@ -439,7 +439,7 @@ packages = []
"apache-airflow-providers-cohere>=1.4.0",
"apache-airflow-providers-common-compat>=1.2.1",
"apache-airflow-providers-common-io>=1.4.2",
- "apache-airflow-providers-common-messaging",
+ "apache-airflow-providers-common-messaging>=1.0.0",
"apache-airflow-providers-common-sql>=1.18.0",
"apache-airflow-providers-databricks>=6.11.0",
"apache-airflow-providers-datadog>=3.8.0",
@@ -453,7 +453,7 @@ packages = []
"apache-airflow-providers-fab>=2.0.0",
"apache-airflow-providers-facebook>=3.7.0",
"apache-airflow-providers-ftp>=3.12.0",
- "apache-airflow-providers-git",
+ "apache-airflow-providers-git>=0.0.1",
"apache-airflow-providers-github>=2.8.0",
"apache-airflow-providers-google>=10.24.0",
"apache-airflow-providers-grpc>=3.7.0",
@@ -473,7 +473,7 @@ packages = []
"apache-airflow-providers-odbc>=4.8.0",
"apache-airflow-providers-openai>=1.5.0",
"apache-airflow-providers-openfaas>=3.7.0",
- "apache-airflow-providers-openlineage>=2.1.2",
+ "apache-airflow-providers-openlineage>=2.1.3",
"apache-airflow-providers-opensearch>=1.5.0",
"apache-airflow-providers-opsgenie>=5.8.0",
"apache-airflow-providers-oracle>=3.12.0",
diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
index 05a8ee3a28c..2ddf4ec83f7 100755
--- a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
+++ b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
@@ -53,9 +53,14 @@ END_PROVIDER_WORKSPACE_MEMBERS = " # End of automatically
generated provider
CUT_OFF_TIMEDELTA = timedelta(days=6 * 30)
+# Temporary override for providers that are not yet included in constraints or
when they need
+# minimum versions for compatibility with Airflow 3
MIN_VERSION_OVERRIDE: dict[str, Version] = {
+ "amazon": parse_version("2.1.3"),
"fab": parse_version("2.0.0"),
- "openlineage": parse_version("2.1.2"),
+ "openlineage": parse_version("2.1.3"),
+ "git": parse_version("0.0.1"),
+ "common.messaging": parse_version("1.0.0"),
}
@@ -86,8 +91,6 @@ all_providers_metadata =
json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
def find_min_provider_version(provider_id: str) -> Version | None:
metadata = all_providers_metadata.get(provider_id)
- if not metadata:
- return None
# We should periodically update the starting date to avoid pip install
resolution issues
# TODO: when min Python version is 3.11 change back the code to
fromisoformat
# https://github.com/apache/airflow/pull/49155/files
@@ -96,21 +99,26 @@ def find_min_provider_version(provider_id: str) -> Version
| None:
)
last_version_newer_than_cutoff: Version | None = None
date_released: datetime | None = None
- versions: list[Version] = sorted([parse_version(version) for version in
metadata])
- for version in reversed(versions):
- provider_info = metadata[str(version)]
- date_released = datetime.strptime(provider_info["date_released"],
"%Y-%m-%dT%H:%M:%SZ").replace(
- tzinfo=timezone.utc
- )
- if date_released < cut_off_date:
- break
- last_version_newer_than_cutoff = version
+ min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
+ if not metadata:
+ if not min_version_override:
+ return None
+ last_version_newer_than_cutoff = min_version_override
+ else:
+ versions: list[Version] = sorted([parse_version(version) for version
in metadata], reverse=True)
+ for version in versions:
+ provider_info = metadata[str(version)]
+ date_released = datetime.strptime(provider_info["date_released"],
"%Y-%m-%dT%H:%M:%SZ").replace(
+ tzinfo=timezone.utc
+ )
+ if date_released < cut_off_date:
+ break
+ last_version_newer_than_cutoff = version
console.print(
f"[bright_blue]Provider id {provider_id} min version found:[/] "
f"{last_version_newer_than_cutoff} (date {date_released}"
)
if last_version_newer_than_cutoff:
- min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
if min_version_override and min_version_override >
last_version_newer_than_cutoff:
console.print(
f"[yellow]Overriding provider id {provider_id} min version:[/]
{min_version_override} "
diff --git a/scripts/in_container/run_generate_constraints.py
b/scripts/in_container/run_generate_constraints.py
index c8160774215..64a4a0806cb 100755
--- a/scripts/in_container/run_generate_constraints.py
+++ b/scripts/in_container/run_generate_constraints.py
@@ -154,7 +154,17 @@ def freeze_packages_to_file(config_params: ConfigParams,
file: TextIO) -> None:
)
count_lines = 0
for line in sorted(result.stdout.split("\n")):
- if line.startswith(("apache_airflow", "apache-airflow==",
"/opt/airflow", "#", "-e")):
+ if line.startswith(
+ (
+ "apache_airflow",
+ "apache-airflow==",
+ "apache-airflow-core==",
+ "apache_airflow_task_sdk=",
+ "/opt/airflow",
+ "#",
+ "-e",
+ )
+ ):
continue
if "@" in line:
continue
@@ -343,6 +353,10 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
find_airflow_core_distributions =
AIRFLOW_DIST_PATH.glob("apache_airflow_core-*.whl")
if find_airflow_core_distributions:
airflow_core_install =
next(find_airflow_core_distributions).as_posix() + "[all]"
+ airflow_task_sdk_install = "./airflow-task-sdk"
+ find_airflow_task_sdk_distribution =
AIRFLOW_DIST_PATH.glob("apache_airflow_task_sdk-*.whl")
+ if find_airflow_task_sdk_distribution:
+ airflow_task_sdk_install =
next(find_airflow_task_sdk_distribution).as_posix()
run_command(
cmd=[
"uv",
@@ -351,7 +365,7 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
"--no-sources",
airflow_install,
airflow_core_install,
- "./task-sdk",
+ airflow_task_sdk_install,
"./airflow-ctl",
"--reinstall", # We need to pull the provider distributions from
PyPI - not use the local ones
*packages_to_install,
diff --git a/task-sdk/pyproject.toml b/task-sdk/pyproject.toml
index 0a267a70d7d..8cc0d195645 100644
--- a/task-sdk/pyproject.toml
+++ b/task-sdk/pyproject.toml
@@ -46,6 +46,7 @@ classifiers = [
]
dependencies = [
+ "apache-airflow-core<=3.1.0,>=3.0.0",
"aiologic>=0.14.0",
"attrs>=24.2.0, !=25.2.0",
"fsspec>=2023.10.0",
@@ -132,14 +133,12 @@ exclude_also = [
[dependency-groups]
codegen = [
- "apache-airflow>=3.0.0",
"datamodel-code-generator[http]==0.28.2",
"svcs>=25.1.0",
"ruff==0.11.2",
"rich>=12.4.4",
]
dev = [
- "apache-airflow",
"apache-airflow-providers-common-sql",
"apache-airflow-providers-standard",
"apache-airflow-devel-common",
diff --git a/task-sdk/src/airflow/sdk/__init__.py
b/task-sdk/src/airflow/sdk/__init__.py
index 8b933942df5..85d4b496f6f 100644
--- a/task-sdk/src/airflow/sdk/__init__.py
+++ b/task-sdk/src/airflow/sdk/__init__.py
@@ -55,7 +55,7 @@ __all__ = [
"teardown",
]
-__version__ = "1.0.0.alpha1"
+__version__ = "1.0.0"
if TYPE_CHECKING:
from airflow.sdk.bases.notifier import BaseNotifier
