This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4e33ceb400 Add option to lower-bind preinstalled provider (#38363)
4e33ceb400 is described below
commit 4e33ceb40023d3d3efa9bb749a97eafe64993c20
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Mar 21 17:27:24 2024 +0100
Add option to lower-bind preinstalled provider (#38363)
We are for now preparing pre-release version of fab for beta
testing of Airflow 2.9 and we should lower-bind the version used
when the packages are build.
---
airflow_pre_installed_providers.txt | 2 +-
hatch_build.py | 26 +++++++++++++++++++++-----
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/airflow_pre_installed_providers.txt
b/airflow_pre_installed_providers.txt
index 8bc3f13546..2d38abfce1 100644
--- a/airflow_pre_installed_providers.txt
+++ b/airflow_pre_installed_providers.txt
@@ -1,7 +1,7 @@
# List of all the providers that are pre-installed when you run `pip install
apache-airflow` without extras
common.io
common.sql
-fab
+fab>=1.0.2dev0
ftp
http
imap
diff --git a/hatch_build.py b/hatch_build.py
index d92587737f..505718058c 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -36,22 +36,38 @@ AIRFLOW_ROOT_PATH = Path(__file__).parent.resolve()
GENERATED_PROVIDERS_DEPENDENCIES_FILE = AIRFLOW_ROOT_PATH / "generated" /
"provider_dependencies.json"
PREINSTALLED_PROVIDERS_FILE = AIRFLOW_ROOT_PATH /
"airflow_pre_installed_providers.txt"
DEPENDENCIES = json.loads(GENERATED_PROVIDERS_DEPENDENCIES_FILE.read_text())
-PREINSTALLED_PROVIDER_IDS = [
+PREINSTALLED_PROVIDER_SPECS = [
package.strip()
for package in PREINSTALLED_PROVIDERS_FILE.read_text().splitlines()
if not package.strip().startswith("#")
]
+
+def get_provider_id(provider_spec: str) -> str:
+ # in case provider_spec is "<provider_id>=<version>"
+ return provider_spec.split(">=")[0]
+
+
+def get_provider_requirement(provider_spec: str) -> str:
+ if ">=" in provider_spec:
+ provider_id, min_version = provider_spec.split(">=")
+ return f"apache-airflow-providers-{provider_id.replace('.',
'-')}>={min_version}"
+ else:
+ return f"apache-airflow-providers-{provider_spec.replace('.', '-')}"
+
+
# if providers are ready, we can preinstall them
PREINSTALLED_PROVIDERS = [
- f"apache-airflow-providers-{provider_id.replace('.','-')}"
- for provider_id in PREINSTALLED_PROVIDER_IDS
- if DEPENDENCIES[provider_id]["state"] == "ready"
+ get_provider_requirement(provider_spec)
+ for provider_spec in PREINSTALLED_PROVIDER_SPECS
+ if DEPENDENCIES[get_provider_id(provider_spec)]["state"] == "ready"
]
+
# if provider is in not-ready or pre-release, we need to install its
dependencies
# however we need to skip apache-airflow itself and potentially any providers
that are
PREINSTALLED_NOT_READY_DEPS = []
-for provider_id in PREINSTALLED_PROVIDER_IDS:
+for provider_spec in PREINSTALLED_PROVIDER_SPECS:
+ provider_id = get_provider_id(provider_spec)
if DEPENDENCIES[provider_id]["state"] not in ["ready", "suspended",
"removed"]:
for dependency in DEPENDENCIES[provider_id]["deps"]:
if dependency.startswith("apache-airflow-providers"):
