This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2a1f17d0521 Add script to move providers to the new directory
structure (#45945)
2a1f17d0521 is described below
commit 2a1f17d0521fd82736c76dfe05d0695505ffffec
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Jan 22 23:43:57 2025 +0100
Add script to move providers to the new directory structure (#45945)
---
.github/boring-cyborg.yml | 4 +-
.../prepare_providers/provider_documentation.py | 22 +-
.../prepare_providers/provider_packages.py | 2 +
.../templates/pyproject_TEMPLATE.toml.jinja2 | 17 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 6 +
dev/moving_providers/README.md | 87 ++++
dev/moving_providers/move_providers.py | 516 +++++++++++++++++++++
dev/requirements.txt | 2 +-
hatch_build.py | 2 +-
providers/airbyte/pyproject.toml | 9 +-
providers/apache/iceberg/pyproject.toml | 10 +-
providers/celery/pyproject.toml | 10 +-
providers/edge/pyproject.toml | 10 +-
scripts/in_container/run_fix_ownership.py | 4 +-
14 files changed, 652 insertions(+), 49 deletions(-)
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index 22cee73155b..2c97219a4e5 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -27,8 +27,8 @@ labelPRBasedOnFilePath:
- providers/tests/alibaba/**/*
- providers/tests/system/alibaba/**/*
- provider:amazon-aws:
- - providers/src/airflow/providers/amazon/aws/**/*
+ provider:amazon:
+ - providers/src/airflow/providers/amazon/**/*
- providers/tests/amazon/aws/**/*
- docs/apache-airflow-providers-amazon/**/*
- providers/tests/system/amazon/aws/**/*
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
index 1fc80a46e8e..6b705148c94 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
@@ -1183,18 +1183,22 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
if in_required_dependencies and line == "]":
in_required_dependencies = False
continue
- if line == "[project.optional-dependencies]":
- in_optional_dependencies = True
- continue
- if in_optional_dependencies and line == "":
- in_optional_dependencies = False
- continue
if line == "[dependency-groups]":
in_dependency_groups = True
continue
if in_dependency_groups and line == "":
in_dependency_groups = False
continue
+ if in_dependency_groups and line.startswith("["):
+ in_dependency_groups = False
+ if line == "[project.optional-dependencies]":
+ in_optional_dependencies = True
+ continue
+ if in_optional_dependencies and line == "":
+ in_optional_dependencies = False
+ continue
+ if in_optional_dependencies and line.startswith("["):
+ in_optional_dependencies = False
if in_required_dependencies:
required_dependencies.append(line)
if in_optional_dependencies:
@@ -1209,7 +1213,9 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
# Add cross-provider dependencies to the optional dependencies if they are
missing
for module in
PROVIDER_DEPENDENCIES.get(provider_details.provider_id)["cross-providers-deps"]:
- if f'"{module}" = [' not in optional_dependencies:
+ if f'"{module}" = [' not in optional_dependencies and
get_pip_package_name(module) not in "\n".join(
+ required_dependencies
+ ):
optional_dependencies.append(f'"{module}" = [')
optional_dependencies.append(f'
"{get_pip_package_name(module)}"')
optional_dependencies.append("]")
@@ -1221,6 +1227,8 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
context=context,
extension=".toml",
autoescape=False,
+ lstrip_blocks=True,
+ trim_blocks=True,
keep_trailing_newline=True,
)
get_pyproject_toml_path.write_text(get_pyproject_toml_content)
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
index 17f00033d21..57ca5942f67 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
@@ -137,6 +137,8 @@ def _prepare_pyproject_toml_file(context: dict[str, Any],
target_path: Path):
context=context,
extension=".toml",
autoescape=False,
+ lstrip_blocks=True,
+ trim_blocks=True,
keep_trailing_newline=True,
)
(target_path / "pyproject.toml").write_text(manifest_content)
diff --git
a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
index 525fa17a2bb..330b0c9f19e 100644
--- a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
@@ -69,25 +69,26 @@ classifiers = [
"Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
+
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
{{ INSTALL_REQUIREMENTS }}
]
+{% if EXTRAS_REQUIREMENTS %}
-{%- if EXTRAS_REQUIREMENTS %}
# The optional dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
[project.optional-dependencies]
{{ EXTRAS_REQUIREMENTS }}
-{%- endif %}
+{% endif %}
+{% if DEPENDENCY_GROUPS %}
-{%- if DEPENDENCY_GROUPS %}
# The dependency groups should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
[dependency-groups]
{{ DEPENDENCY_GROUPS }}
-{%- endif %}
+{% endif %}
[project.urls]
"Documentation" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME
}}/{{RELEASE}}"
@@ -100,13 +101,13 @@ dependencies = [
[project.entry-points."apache_airflow_provider"]
provider_info = "airflow.providers.{{ PROVIDER_ID
}}.get_provider_info:get_provider_info"
+{% if PLUGINS %}
-{%- if PLUGINS %}
[project.entry-points."airflow.plugins"]
-{%- for plugin in PLUGINS %}
+{% for plugin in PLUGINS %}
{{ plugin.name }} = "{{ plugin.package_name }}:{{ plugin.class_name }}"
-{%- endfor %}
-{%- endif %}
+{% endfor %}
+{% endif %}
[tool.flit.module]
name = "airflow.providers.{{ PROVIDER_ID }}"
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 46f367618b5..fec5ed898d6 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -834,6 +834,8 @@ def render_template(
context: dict[str, Any],
extension: str,
autoescape: bool = True,
+ lstrip_blocks: bool = False,
+ trim_blocks: bool = False,
keep_trailing_newline: bool = False,
) -> str:
"""
@@ -842,6 +844,8 @@ def render_template(
:param context: Jinja2 context
:param extension: Target file extension
:param autoescape: Whether to autoescape HTML
+ :param lstrip_blocks: Whether to strip leading blocks
+ :param trim_blocks: Whether to trim blocks
:param keep_trailing_newline: Whether to keep the newline in rendered
output
:return: rendered template
"""
@@ -852,6 +856,8 @@ def render_template(
loader=template_loader,
undefined=jinja2.StrictUndefined,
autoescape=autoescape,
+ lstrip_blocks=lstrip_blocks,
+ trim_blocks=trim_blocks,
keep_trailing_newline=keep_trailing_newline,
)
template =
template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
diff --git a/dev/moving_providers/README.md b/dev/moving_providers/README.md
new file mode 100644
index 00000000000..abd885b69f3
--- /dev/null
+++ b/dev/moving_providers/README.md
@@ -0,0 +1,87 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update
-->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents** *generated with
[DocToc](https://github.com/thlorenz/doctoc)*
+
+- [Moving providers to new structure](#moving-providers-to-new-structure)
+ - [How to use the script](#how-to-use-the-script)
+ - [Options](#options)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# Moving providers to new structure
+
+We are moving providers to a new structure, where each provider has a separate
sub-project in
+"providers" sub-folder.
+
+This means that we need to migrate some 90+ providers to the new structure.
This is a big task and while we
+could do it in one huge PR, it would be disruptive and likely take some time
to review and fix some individual
+edge-cases - even if we have automated most of the work.
+
+This directory contains a script that contributors can use to move a provider
(or a few providers to the
+new structure as a starting point for their PR. Most of the work is automated,
but there will be likely
+some manual adjustments needed in more complex cases.
+
+## How to use the script
+
+The script follows https://peps.python.org/pep-0723/ and uses inlined
dependencies - so it can be run as-is
+by modern tools without creating dedicated virtualenv - the virtualenv with
dependencies is
+created on-the-fly by PEP 723 compatible tools. For example one can use uv to
run it:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba
+```
+
+## Options
+
+
+> [!NOTE]
+> You can see all the options by running the script with `--help` option:
+>
+> ```shell
+> uv run dev/moving_providers/move_providers.py --help
+> ```
+
+By default the script runs in `--dry-run` mode, which means it will not make
any changes to the file system,
+but will print what it would do. To actually move the files, you need to pass
`--no-dry-run` option and you
+will be asked to commit the code and create a PR:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --no-dry-run
+```
+
+You can specify multiple providers to move in one go:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba amazon microsoft.azure
+```
+
+You can specify `--verbose` option to see more details about what the script
is doing:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --verbose
+```
+
+You can also specify `--quiet` option to see less output:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --quiet
+```
diff --git a/dev/moving_providers/move_providers.py
b/dev/moving_providers/move_providers.py
new file mode 100755
index 00000000000..2a4316e2a54
--- /dev/null
+++ b/dev/moving_providers/move_providers.py
@@ -0,0 +1,516 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+# "click>=8.1.8",
+# "rich>=13.6.0",
+# "rich-click>=1.7.1",
+# "pyyaml>=6.0.1",
+# ]
+# ///
+from __future__ import annotations
+
+import difflib
+import shutil
+import subprocess
+import sys
+from functools import cache
+from pathlib import Path
+
+import rich_click as click
+from rich.console import Console
+from rich.syntax import Syntax
+
+ROOT_PROJECT_DIR_PATH = Path(__file__).parent.parent.parent
+PROVIDERS_DIR_PATH = ROOT_PROJECT_DIR_PATH / "providers"
+OLD_PROVIDERS_SRC_DIR_PATH = PROVIDERS_DIR_PATH / "src"
+OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH = OLD_PROVIDERS_SRC_DIR_PATH
/ "airflow" / "providers"
+OLD_PROVIDERS_TEST_DIR_PATH = ROOT_PROJECT_DIR_PATH / "providers" / "tests"
+OLD_PROVIDERS_SYSTEM_TEST_DIR_PATH = OLD_PROVIDERS_TEST_DIR_PATH / "system"
+DOCS_DIR_PATH = ROOT_PROJECT_DIR_PATH / "docs"
+
+
+@cache
+def _get_all_old_providers() -> list[str]:
+ return sorted(
+ [
+ ".".join(
+
provider_yaml_path.parent.relative_to(OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH).parts
+ )
+ for provider_yaml_path in
OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH.rglob("provider.yaml")
+ ]
+ )
+
+
+def _get_provider_distribution_name(provider_id: str) -> str:
+ return f"apache-airflow-providers-{provider_id.replace('.', '-')}"
+
+
+def _get_provider_only_path(provider_id: str) -> str:
+ return provider_id.replace(".", "/")
+
+
+CONTENT_OVERRIDE = ["This content will be overridden by pre-commit hook"]
+
+console = Console(color_system="standard")
+
+is_verbose = False
+is_quiet = False
+is_dry_run = False
+
+
+def _do_stuff(
+ *,
+ syntax: str,
+ from_path: Path | None = None,
+ to_path: Path | None = None,
+ from_content: list[str] | None = None,
+ updated_content: list[str] | None = None,
+ delete_from: bool = False,
+):
+ if not to_path:
+ # in place update
+ to_path = from_path
+ updated_str = ""
+ if updated_content:
+ updated_str = "\n".join(updated_content) + "\n"
+ if is_verbose:
+ console.print(Syntax(updated_str, syntax, theme="ansi_dark"))
+ console.rule()
+ if not is_quiet:
+ if updated_content and from_content and from_path and to_path:
+ diff = difflib.unified_diff(
+ from_content, updated_content, fromfile=from_path.as_posix(),
tofile=to_path.as_posix()
+ )
+ console.print(Syntax("\n".join(diff), "diff", theme="ansi_dark"))
+ console.print()
+ elif updated_content and not from_content and to_path:
+ console.print(Syntax(updated_str, syntax, theme="ansi_dark"))
+ elif updated_content and to_path:
+ console.print(f"\n[yellow]Creating {to_path}\n")
+ elif not from_content and not updated_content and from_path and
to_path and delete_from:
+ console.print(f"\n[yellow]Moving[/] {from_path} -> {to_path}\n")
+ elif not from_content and not updated_content and from_path and
to_path and not delete_from:
+ console.print(f"\n[yellow]Copying[/] {from_path} -> {to_path}\n")
+ elif delete_from and from_path:
+ console.print(f"\n[yellow]Deleting {from_path}\n")
+ if not is_dry_run:
+ if updated_content and to_path:
+ to_path.parent.mkdir(parents=True, exist_ok=True)
+ to_path.write_text(updated_str)
+ console.print(f"\n[yellow]Written {to_path}\n")
+ elif not from_content and not updated_content and from_path and
to_path:
+ if delete_from:
+ to_path.parent.mkdir(parents=True, exist_ok=True)
+ if from_path.is_dir() and to_path.exists():
+ shutil.rmtree(to_path)
+ shutil.move(from_path, to_path)
+ console.print(f"\n[yellow]Moved {from_path} -> {to_path}\n")
+ return
+ else:
+ to_path.parent.mkdir(parents=True, exist_ok=True)
+ if from_path.is_dir():
+ shutil.rmtree(to_path)
+ shutil.copytree(from_path, to_path)
+ else:
+ to_path.write_text(from_path.read_text())
+ console.print(f"\n[yellow]Copied {from_path} -> {to_path}\n")
+ return
+ if delete_from and from_path:
+ from_path.unlink()
+ console.print(f"\n[yellow]Deleted {from_path}\n")
+
+
[email protected]()
[email protected]("provider_ids", type=click.Choice(_get_all_old_providers()),
required=True, nargs=-1)
[email protected](
+ "--dry-run/--no-dry-run",
+ default=True,
+ help="Whether to run the command without making changes.",
+ show_default=True,
+ is_flag=True,
+)
[email protected](
+ "--verbose",
+ help="Whether to show complete content of generated files. (mutually
exclusive with --quiet).",
+ is_flag=True,
+)
[email protected](
+ "--skip-build-file-generation",
+ help="When set, the step to generate build files is skipped.",
+ is_flag=True,
+)
[email protected](
+ "--quiet",
+ help="Whether to be quite - only show providers updated (mutually
exclusive with --verbose).",
+ is_flag=True,
+)
+def move_providers(
+ provider_ids: tuple[str, ...], dry_run: bool, skip_build_file_generation:
bool, verbose: bool, quiet: bool
+):
+ if quiet and verbose:
+ console.print("\n[red]Cannot use --quiet and --verbose at the same
time\n")
+ sys.exit(1)
+ if dry_run:
+ console.print("\n[yellow]Running in dry-run mode, no changes will be
made\n")
+ global is_quiet, is_verbose, is_dry_run
+ is_quiet = quiet
+ is_verbose = verbose
+ is_dry_run = dry_run
+
+ console.print("\n[blue]Moving providers:[/]\n")
+ console.print("* " + "\n *".join(provider_ids))
+ console.print()
+
+ for provider_id in provider_ids:
+ console.rule(f"\n[magenta]Moving provider: {provider_id}[/]\n",
align="left")
+ move_provider(provider_id)
+ console.rule()
+ console.print()
+
+ count_providers = len(_get_all_old_providers())
+ if not dry_run:
+ subprocess.run("git add .", shell=True, check=True)
+ if not skip_build_file_generation:
+ subprocess.run("pre-commit run update-providers-build-files",
shell=True, check=False)
+ subprocess.run("breeze ci-image build --python 3.9 --answer yes",
shell=True, check=True)
+ subprocess.run("git add . ", shell=True, check=False)
+ subprocess.run("git diff HEAD", shell=True, check=False)
+ console.print("\n[bright_green]First part of migration is
complete[/].\n")
+ console.print("[yellow]Next steps:")
+ console.print("* run `pre-commit run`")
+ console.print("* fix all remaining errors, ")
+ console.print("* create branch, commit the changes and create a PR!\n")
+ console.print(
+ f"\nAfter the PR is merged there will be {count_providers -
len(provider_ids)} providers "
+ f"left in the old location.\n"
+ )
+ else:
+ console.print("\n[yellow]Dry-run mode, no changes were made.\n")
+ console.print(f"\nThere are currently {count_providers} providers left in
the old structure.\n")
+
+
+def fix_boring_cyborg(provider_id: str):
+ boring_cyborg_file_path = ROOT_PROJECT_DIR_PATH / ".github" /
"boring-cyborg.yml"
+ console.print(f"\n[bright_blue]Updating {boring_cyborg_file_path}\n")
+ original_content = boring_cyborg_file_path.read_text().splitlines()
+ updated_content = []
+ in_provider = False
+ for line in original_content:
+ if not in_provider:
+ updated_content.append(line)
+ if line.strip() == f"provider:{provider_id.replace('.', '-')}:":
+ in_provider = True
+ updated_content.append(f" - providers/{provider_id.replace('.',
'/')}/**")
+ updated_content.append("")
+ if in_provider and line.strip() == "":
+ in_provider = False
+ _do_stuff(
+ syntax="yaml",
+ from_path=boring_cyborg_file_path,
+ from_content=original_content,
+ updated_content=updated_content,
+ )
+
+
+def add_docs_to_gitignore(provider_id: str):
+ gitignore_path = DOCS_DIR_PATH / ".gitignore"
+ console.print(f"\n[bright_blue]Updating {gitignore_path}\n")
+ original_content = gitignore_path.read_text().splitlines()
+ provider_line = f"apache-airflow-providers-{provider_id.replace('.', '-')}"
+ if provider_line in original_content:
+ console.print(f"\n[yellow]Provider {provider_id} already in
.gitignore\n")
+ return
+ updated_content = []
+ updated = False
+ for line in original_content:
+ if not line.startswith("#") and line > provider_line and not updated:
+ updated_content.append(provider_line)
+ updated = True
+ updated_content.append(line)
+ if not updated:
+ updated_content.append(provider_line)
+ _do_stuff(
+ syntax="gitignore",
+ from_path=gitignore_path,
+ from_content=original_content,
+ updated_content=updated_content,
+ )
+
+
+def _replace_string(path: Path, old: str, new: str):
+ content = path.read_text()
+ new_content = content.replace(old, new)
+ if content != new_content:
+ console.print(f"\n[bright_blue]Replacing {old} with {new} in {path}\n")
+ if not is_dry_run:
+ path.write_text(new_content)
+
+
+def remove_changelog(provider_id: str):
+ changelog_path = DOCS_DIR_PATH /
_get_provider_distribution_name(provider_id) / "changelog.rst"
+ console.print(f"\n[bright_blue]Deleting {changelog_path}\n")
+ _do_stuff(syntax="gitignore", from_path=changelog_path, delete_from=True)
+
+
+def create_readme(provider_id: str):
+ readme_path = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) /
"README.rst"
+ console.print(f"\n[bright_blue]Creating {readme_path}\n")
+ _do_stuff(syntax="rst", to_path=readme_path,
updated_content=CONTENT_OVERRIDE)
+
+
+def move_docs(provider_id: str):
+ source_doc_dir = DOCS_DIR_PATH /
_get_provider_distribution_name(provider_id)
+ dest_doc_dir = PROVIDERS_DIR_PATH / _get_provider_only_path(provider_id) /
"docs"
+ console.print(f"\n[bright_blue]Moving docs to {dest_doc_dir}\n")
+ _do_stuff(syntax="rst", from_path=source_doc_dir, to_path=dest_doc_dir,
delete_from=True)
+ provider_package_source_dir =
OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH / _get_provider_only_path(
+ provider_id
+ )
+ _do_stuff(
+ syntax="rst",
+ from_path=provider_package_source_dir / "CHANGELOG.rst",
+ to_path=dest_doc_dir / "changelog.rst",
+ delete_from=True,
+ )
+ _do_stuff(
+ syntax="txt",
+ from_path=provider_package_source_dir / ".latest-doc-only-change.txt",
+ to_path=dest_doc_dir / ".latest-doc-only-change.txt",
+ delete_from=True,
+ )
+
+
+def move_provider_yaml(provider_id: str) -> tuple[list[str], list[str],
list[str]]:
+ source_provider_yaml_path = (
+ OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH
+ / _get_provider_only_path(provider_id)
+ / "provider.yaml"
+ )
+ target_provider_yaml_path = PROVIDERS_DIR_PATH /
_get_provider_only_path(provider_id) / "provider.yaml"
+ console.print(f"\n[bright_blue]Moving {source_provider_yaml_path} to
{target_provider_yaml_path}\n")
+ original_content = source_provider_yaml_path.read_text().splitlines()
+ in_dependencies = False
+ in_optional_dependencies = False
+ in_devel_dependencies = False
+ updated_content = []
+
+ dependencies = []
+ optional_dependencies = []
+ devel_dependencies = []
+ for line in original_content:
+ if line == "dependencies:" and not in_dependencies:
+ in_dependencies = True
+ continue
+ if in_dependencies:
+ if not line:
+ continue
+ if line.startswith(" -"):
+ dependencies.append(f' "{line[len(" - ") :]}",')
+ elif line.strip().startswith("#"):
+ dependencies.append(f" {line.strip()}")
+ else:
+ in_dependencies = False
+ if line == "devel-dependencies:" and not in_devel_dependencies:
+ in_devel_dependencies = True
+ continue
+ if in_devel_dependencies:
+ if not line:
+ continue
+ if line.startswith(" - "):
+ devel_dependencies.append(f' "{line[len(" - ") :]}",')
+ elif line.strip().startswith("#"):
+ devel_dependencies.append(f" {line.strip()}")
+ else:
+ in_devel_dependencies = False
+ if line == "additional-extras:" and not in_optional_dependencies:
+ in_optional_dependencies = True
+ continue
+ if in_optional_dependencies:
+ if not line:
+ continue
+ if line.startswith(" "):
+ optional_dependencies.append(line)
+ else:
+ in_optional_dependencies = False
+ if not in_dependencies and not in_optional_dependencies and not
in_devel_dependencies:
+ updated_content.append(line)
+
+ _do_stuff(
+ syntax="yml",
+ from_path=source_provider_yaml_path,
+ to_path=target_provider_yaml_path,
+ from_content=original_content,
+ updated_content=updated_content,
+ delete_from=True,
+ )
+ if optional_dependencies:
+ in_dependency = False
+ optional_dependencies_processed = []
+ for line in optional_dependencies:
+ if line.startswith(" - name: "):
+ name = line[len(" - name: ") :]
+ if in_dependency:
+ optional_dependencies_processed.append("]")
+ optional_dependencies_processed.append(f'"{name}" = [')
+ in_dependency = True
+ elif line.startswith(" -"):
+ dependency = line[len(" - ") :]
+ optional_dependencies_processed.append(f' "{dependency}",')
+ elif line.startswith(" #"):
+ optional_dependencies_processed.append(f" {line.strip()}")
+ elif line.startswith(" #"):
+ if in_dependency:
+ optional_dependencies_processed.append("]")
+ in_dependency = False
+ optional_dependencies_processed.append(f"{line.strip()}")
+ optional_dependencies_processed.append("]")
+ else:
+ optional_dependencies_processed = []
+ return (
+ dependencies,
+ devel_dependencies,
+ optional_dependencies_processed,
+ )
+
+
+def create_pyproject_toml(
+ provider_id: str,
+ dependencies: list[str],
+ devel_dependencies: list[str],
+ optional_dependencies: list[str],
+):
+ dependencies_str = "\n".join(dependencies)
+ devel_dependencies_str = "\n".join(devel_dependencies)
+ optional_dependencies_str = "\n".join(optional_dependencies)
+ start_pyproject_toml = f"""
+# Content of this file will be replaced by pre-commit hook
+[build-system]
+requires = ["flit_core==3.10.1"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-SOME_PROVIDER"
+version = "VERSION"
+description = "Provider package PROVIDER for Apache Airflow"
+readme = "README.rst"
+
+dependencies = [
+{dependencies_str}
+]
+"""
+ optional_dependencies_toml = f"""
+[project.optional-dependencies]
+{optional_dependencies_str}
+"""
+ devel_dependencies_toml = f"""
+[dependency-groups]
+dev = [
+{devel_dependencies_str}
+]
+
+[project.urls]
+"""
+ pyproject_toml_path = PROVIDERS_DIR_PATH /
_get_provider_only_path(provider_id) / "pyproject.toml"
+ console.print(
+ f"\n[bright_blue]Creating basic pyproject.toml for {provider_id} in
{pyproject_toml_path}\n"
+ )
+
+ pyproject_toml_content = start_pyproject_toml
+ if optional_dependencies:
+ pyproject_toml_content += optional_dependencies_toml
+ if devel_dependencies:
+ pyproject_toml_content += devel_dependencies_toml
+
+ _do_stuff(syntax="toml", to_path=pyproject_toml_path,
updated_content=pyproject_toml_content.splitlines())
+
+
+def move_sources(provider_id: str):
+ source_provider_dir = OLD_PROVIDERS_AIRFLOW_PROVIDERS_SRC_PACKAGE_PATH /
_get_provider_only_path(
+ provider_id
+ )
+ dest_provider_dir = (
+ PROVIDERS_DIR_PATH
+ / _get_provider_only_path(provider_id)
+ / "src"
+ / "airflow"
+ / "providers"
+ / _get_provider_only_path(provider_id)
+ )
+ console.print(f"\n[bright_blue]Moving sources from {source_provider_dir}
to {dest_provider_dir}\n")
+ _do_stuff(syntax="bash", from_path=source_provider_dir,
to_path=dest_provider_dir, delete_from=True)
+
+
+def move_tests(provider_id: str):
+ source_test_dir = OLD_PROVIDERS_TEST_DIR_PATH /
_get_provider_only_path(provider_id)
+ dest_test_dir = (
+ PROVIDERS_DIR_PATH
+ / _get_provider_only_path(provider_id)
+ / "tests"
+ / "providers"
+ / _get_provider_only_path(provider_id)
+ )
+ console.print(f"\n[bright_blue]Moving tests from {source_test_dir} to
{dest_test_dir}\n")
+ _do_stuff(syntax="bash", from_path=source_test_dir, to_path=dest_test_dir,
delete_from=True)
+
+
+def move_system_tests(provider_id: str):
+ source_system_test_dir = OLD_PROVIDERS_SYSTEM_TEST_DIR_PATH /
_get_provider_only_path(provider_id)
+ dest_system_test_dir = (
+ PROVIDERS_DIR_PATH
+ / _get_provider_only_path(provider_id)
+ / "tests"
+ / "system"
+ / _get_provider_only_path(provider_id)
+ )
+ console.print(
+ f"\n[bright_blue]Moving system tests from {source_system_test_dir} to
{dest_system_test_dir}\n"
+ )
+ _do_stuff(syntax="bash", from_path=source_system_test_dir,
to_path=dest_system_test_dir, delete_from=True)
+
+
+def replace_system_test_example_includes(provider_id: str):
+ target_doc_providers_dir = PROVIDERS_DIR_PATH /
_get_provider_only_path(provider_id) / "docs"
+ console.print(f"\n[bright_blue]Replacing system test example includes in
{target_doc_providers_dir}\n")
+ for rst_file in target_doc_providers_dir.rglob("*.rst"):
+ provider_only_path = _get_provider_only_path(provider_id)
+ _replace_string(
+ rst_file,
+ f"../providers/tests/system/{provider_only_path}/",
+
f"../providers/{provider_only_path}/tests/system/{provider_only_path}/",
+ )
+
+
+def move_provider(provider_id: str):
+ fix_boring_cyborg(provider_id)
+ add_docs_to_gitignore(provider_id)
+ remove_changelog(provider_id)
+ create_readme(provider_id)
+ move_docs(provider_id)
+ dependencies, devel_dependencies, optional_dependencies =
move_provider_yaml(provider_id)
+ create_pyproject_toml(provider_id, dependencies, devel_dependencies,
optional_dependencies)
+ move_sources(provider_id)
+ move_tests(provider_id)
+ move_system_tests(provider_id)
+ replace_system_test_example_includes(provider_id)
+
+
+if __name__ == "__main__":
+ move_providers()
diff --git a/dev/requirements.txt b/dev/requirements.txt
index a631eb34cc6..ac47dd8b745 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,4 +1,4 @@
-click>=8.0
+click>=8.1.8
jinja2>=2.11.3
keyring==25.6.0
PyGithub
diff --git a/hatch_build.py b/hatch_build.py
index baa6958c336..cb0309c9429 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -189,7 +189,7 @@ DEVEL_EXTRAS: dict[str, list[str]] = {
"pdbr>=0.8.9",
],
"devel-devscripts": [
- "click>=8.0",
+ "click>=8.1.8",
"gitpython>=3.1.40",
"incremental>=24.7.2",
"pipdeptree>=2.13.1",
diff --git a/providers/airbyte/pyproject.toml b/providers/airbyte/pyproject.toml
index 8a49050f021..37022d69b5c 100644
--- a/providers/airbyte/pyproject.toml
+++ b/providers/airbyte/pyproject.toml
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -44,14 +43,10 @@ classifiers = [
"Intended Audience :: System Administrators",
"Framework :: Apache Airflow",
"Framework :: Apache Airflow :: Provider",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Topic :: System :: Monitoring",
+ "License :: OSI Approved :: Apache Software License", "Programming
Language :: Python :: 3.9", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python ::
3.12", "Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
+
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
diff --git a/providers/apache/iceberg/pyproject.toml
b/providers/apache/iceberg/pyproject.toml
index cfc1cf1023d..8059b1b554d 100644
--- a/providers/apache/iceberg/pyproject.toml
+++ b/providers/apache/iceberg/pyproject.toml
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -44,19 +43,16 @@ classifiers = [
"Intended Audience :: System Administrators",
"Framework :: Apache Airflow",
"Framework :: Apache Airflow :: Provider",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Topic :: System :: Monitoring",
+ "License :: OSI Approved :: Apache Software License", "Programming
Language :: Python :: 3.9", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python ::
3.12", "Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
+
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
"apache-airflow>=2.9.0",
]
+
# The dependency groups should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
[dependency-groups]
diff --git a/providers/celery/pyproject.toml b/providers/celery/pyproject.toml
index 6d510f852e1..3ec3c28f277 100644
--- a/providers/celery/pyproject.toml
+++ b/providers/celery/pyproject.toml
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -44,14 +43,10 @@ classifiers = [
"Intended Audience :: System Administrators",
"Framework :: Apache Airflow",
"Framework :: Apache Airflow :: Provider",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Topic :: System :: Monitoring",
+ "License :: OSI Approved :: Apache Software License", "Programming
Language :: Python :: 3.9", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python ::
3.12", "Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
+
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
@@ -65,6 +60,7 @@ dependencies = [
"flower>=1.0.0",
"google-re2>=1.0",
]
+
# The optional dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
[project.optional-dependencies]
diff --git a/providers/edge/pyproject.toml b/providers/edge/pyproject.toml
index dfe14b79272..055e38ec525 100644
--- a/providers/edge/pyproject.toml
+++ b/providers/edge/pyproject.toml
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -44,14 +43,10 @@ classifiers = [
"Intended Audience :: System Administrators",
"Framework :: Apache Airflow",
"Framework :: Apache Airflow :: Provider",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Topic :: System :: Monitoring",
+ "License :: OSI Approved :: Apache Software License", "Programming
Language :: Python :: 3.9", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python ::
3.12", "Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
+
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
@@ -71,6 +66,7 @@ dependencies = [
[project.entry-points."apache_airflow_provider"]
provider_info = "airflow.providers.edge.get_provider_info:get_provider_info"
+
[project.entry-points."airflow.plugins"]
edge_executor =
"airflow.providers.edge.plugins.edge_executor_plugin:EdgeExecutorPlugin"
diff --git a/scripts/in_container/run_fix_ownership.py
b/scripts/in_container/run_fix_ownership.py
index 8062a72980d..bb25fc17bdc 100755
--- a/scripts/in_container/run_fix_ownership.py
+++ b/scripts/in_container/run_fix_ownership.py
@@ -77,8 +77,8 @@ def change_ownership_of_files(path: Path) -> None:
# another place
if os.environ.get("VERBOSE_COMMANDS", "false") == "true":
print(f"Could not change ownership of {file}")
- if count_files:
- print(f"Changed ownership of {count_files} files back to
{host_user_id}:{host_group_id}.")
+ if count_files and os.environ.get("VERBOSE_COMMANDS", "false") == "true":
+ print(f"Changed ownership of {count_files} files back to
{host_user_id}:{host_group_id}.")
if __name__ == "__main__":
