This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4280b83977c Move apache.iceberg provider to new providers structure
(#45809)
4280b83977c is described below
commit 4280b83977cd5a53c2b24143f3c9a6a63e298acc
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Jan 21 17:26:53 2025 +0100
Move apache.iceberg provider to new providers structure (#45809)
This also add the feature of managing devel-dependencies via
dependency groups in pyproject.toml and adjust tests to cover
case where providers can be two levels deeper.
---
.github/boring-cyborg.yml | 5 +-
.../airflow_breeze/commands/developer_commands.py | 14 +-
.../prepare_providers/provider_documentation.py | 11 ++
.../templates/pyproject_TEMPLATE.toml.jinja2 | 7 +
dev/breeze/src/airflow_breeze/utils/packages.py | 6 +
.../tests/test_pytest_args_for_test_types.py | 10 +-
docs/.gitignore | 1 +
.../changelog.rst | 25 ---
docs/conf.py | 6 +
docs/exts/provider_yaml_utils.py | 14 +-
providers/airbyte/docs/index.rst | 4 +-
providers/airbyte/docs/operators/airbyte.rst | 4 +-
providers/airbyte/provider.yaml | 2 +-
.../airflow/providers/airbyte/get_provider_info.py | 2 +-
.../system/{providers => }/airbyte/__init__.py | 0
.../airbyte/example_airbyte_trigger_job.py | 0
providers/apache/iceberg/README.rst | 61 +++++++
.../iceberg/docs}/.latest-doc-only-change.txt | 0
.../iceberg/docs/changelog.rst} | 0
.../apache/iceberg/docs}/commits.rst | 0
.../apache/iceberg/docs}/connections.rst | 0
.../apache/iceberg/docs}/index.rst | 2 +-
.../docs}/installing-providers-from-sources.rst | 0
.../iceberg/docs/integration-logo}/iceberg.png | Bin
.../apache/iceberg/docs}/security.rst | 0
.../providers => }/apache/iceberg/provider.yaml | 8 +-
.../apache/iceberg/pyproject.toml | 61 ++-----
.../src/airflow/providers/apache/iceberg/LICENSE | 201 +++++++++++++++++++++
.../airflow/providers/apache/iceberg/__init__.py | 0
.../providers/apache/iceberg/get_provider_info.py | 54 ++++++
.../providers/apache/iceberg/hooks/__init__.py | 0
.../providers/apache/iceberg/hooks/iceberg.py | 0
.../iceberg/tests/conftest.py} | 16 ++
.../iceberg/tests/providers}/__init__.py | 3 +
.../iceberg/tests/providers/apache}/__init__.py | 3 +
.../tests/providers}/apache/iceberg/__init__.py | 0
.../providers}/apache/iceberg/hooks/__init__.py | 0
.../apache/iceberg/hooks/test_iceberg.py | 0
.../tests/system/apache/iceberg/__init__.py | 0
.../tests/system/apache/iceberg/example_iceberg.py | 0
pyproject.toml | 11 +-
.../ci/pre_commit/update_providers_dependencies.py | 7 +-
42 files changed, 434 insertions(+), 104 deletions(-)
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index 98f850b3b03..f7a87d16a54 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -74,10 +74,7 @@ labelPRBasedOnFilePath:
- providers/tests/system/apache/hive/**/*
provider:apache-iceberg:
- - providers/src/airflow/providers/apache/iceberg/**/*
- - docs/apache-airflow-providers-apache-iceberg/**/*
- - providers/tests/apache/iceberg/**/*
- - providers/tests/system/apache/iceberg/**/*
+ - providers/apache/iceberg/**
provider:apache-impala:
- providers/src/airflow/providers/apache/impala/**/*
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 07afbb009b6..0e3c01ca978 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -722,11 +722,15 @@ def build_docs(
)
rebuild_or_pull_ci_image_if_needed(command_params=build_params)
if clean_build:
- docs_dir = AIRFLOW_SOURCES_ROOT / "docs"
- for dir_name in ["_build", "_doctrees", "_inventory_cache", "_api"]:
- for directory in docs_dir.rglob(dir_name):
- get_console().print(f"[info]Removing {directory}")
- shutil.rmtree(directory, ignore_errors=True)
+ directories_to_clean = ["_build", "_doctrees", "_inventory_cache",
"_api"]
+ else:
+ directories_to_clean = ["_api"]
+ docs_dir = AIRFLOW_SOURCES_ROOT / "docs"
+ for dir_name in directories_to_clean:
+ for directory in docs_dir.rglob(dir_name):
+ get_console().print(f"[info]Removing {directory}")
+ shutil.rmtree(directory, ignore_errors=True)
+
docs_list_as_tuple: tuple[str, ...] = ()
if package_list and len(package_list):
get_console().print(f"\n[info]Populating provider list from
PACKAGE_LIST env as {package_list}")
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
index ce46aabb1b7..1fc80a46e8e 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
@@ -1172,8 +1172,10 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
pyproject_toml_content = get_pyproject_toml_path.read_text()
required_dependencies: list[str] = []
optional_dependencies: list[str] = []
+ dependency_groups: list[str] = []
in_required_dependencies = False
in_optional_dependencies = False
+ in_dependency_groups = False
for line in pyproject_toml_content.splitlines():
if line == "dependencies = [":
in_required_dependencies = True
@@ -1187,10 +1189,18 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
if in_optional_dependencies and line == "":
in_optional_dependencies = False
continue
+ if line == "[dependency-groups]":
+ in_dependency_groups = True
+ continue
+ if in_dependency_groups and line == "":
+ in_dependency_groups = False
+ continue
if in_required_dependencies:
required_dependencies.append(line)
if in_optional_dependencies:
optional_dependencies.append(line)
+ if in_dependency_groups:
+ dependency_groups.append(line)
# For additional providers we want to load the dependencies and see if
cross-provider-dependencies are
# present and if not, add them to the optional dependencies
@@ -1204,6 +1214,7 @@ def _regenerate_pyproject_toml(context: dict[str, Any],
provider_details: Provid
optional_dependencies.append(f'
"{get_pip_package_name(module)}"')
optional_dependencies.append("]")
context["EXTRAS_REQUIREMENTS"] = "\n".join(optional_dependencies)
+ context["DEPENDENCY_GROUPS"] = "\n".join(dependency_groups)
get_pyproject_toml_content = render_template(
template_name="pyproject",
diff --git
a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
index b3a6ad8871b..525fa17a2bb 100644
--- a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
@@ -82,6 +82,13 @@ dependencies = [
{{ EXTRAS_REQUIREMENTS }}
{%- endif %}
+{%- if DEPENDENCY_GROUPS %}
+# The dependency groups should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[dependency-groups]
+{{ DEPENDENCY_GROUPS }}
+{%- endif %}
+
[project.urls]
"Documentation" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME
}}/{{RELEASE}}"
"Changelog" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME
}}/{{RELEASE}}/changelog.html"
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 538abc35a23..06b7ef4dfdb 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -187,6 +187,10 @@ def
refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path):
optional_dependencies =
toml_content["project"].get("optional-dependencies")
if optional_dependencies:
PROVIDER_METADATA[provider_id]["optional-dependencies"] =
optional_dependencies
+ dependency_groups = toml_content.get("dependency-groups")
+ if dependency_groups and dependency_groups.get("dev"):
+ devel_dependencies = dependency_groups.get("dev")
+ PROVIDER_METADATA[provider_id]["devel-dependencies"] =
devel_dependencies
def clear_cache_for_provider_metadata(provider_yaml_path: Path):
@@ -806,6 +810,8 @@ def get_provider_jinja_context(
"EXTRAS_REQUIREMENTS": get_package_extras_for_old_providers(
provider_id=provider_details.provider_id,
version_suffix=version_suffix
),
+ # TODO(potiuk) - remove when all providers are new-style
+ "DEPENDENCY_GROUPS": {},
"CHANGELOG_RELATIVE_PATH": os.path.relpath(
provider_details.root_provider_path,
provider_details.documentation_provider_package_path,
diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py
b/dev/breeze/tests/test_pytest_args_for_test_types.py
index 1dffaffb1cc..4fb059979d1 100644
--- a/dev/breeze/tests/test_pytest_args_for_test_types.py
+++ b/dev/breeze/tests/test_pytest_args_for_test_types.py
@@ -26,9 +26,13 @@ from airflow_breeze.utils.run_tests import
convert_parallel_types_to_folders, co
# TODO(potiuk): rename to all_providers when we move all providers to the new
structure
def _all_new_providers() -> list[str]:
all_new_providers: list[str] = []
- for file in (AIRFLOW_SOURCES_ROOT / "providers").iterdir():
- if file.is_dir() and not file.name.startswith(".") and file.name not
in ["src", "tests"]:
- all_new_providers.append(file.name)
+ providers_root = AIRFLOW_SOURCES_ROOT / "providers"
+ for file in providers_root.rglob("provider.yaml"):
+ # TODO: remove this check when all providers are moved to the new
structure
+ if file.is_relative_to(providers_root / "src"):
+ continue
+ provider_path = file.parent.relative_to(providers_root)
+ all_new_providers.append(provider_path.as_posix())
return sorted(all_new_providers)
diff --git a/docs/.gitignore b/docs/.gitignore
index 114b19b7e9f..97a0827b426 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1,5 +1,6 @@
# TODO(potiuk): change it to apache-airflow-providers-* after all providers
are migrated to the new structure
# Eventually when we swtich to individually build docs for each provider, we
should remove this altogether
apache-airflow-providers-airbyte
+apache-airflow-providers-apache-iceberg
apache-airflow-providers-celery
apache-airflow-providers-edge
diff --git a/docs/apache-airflow-providers-apache-iceberg/changelog.rst
b/docs/apache-airflow-providers-apache-iceberg/changelog.rst
deleted file mode 100644
index 220a77a762b..00000000000
--- a/docs/apache-airflow-providers-apache-iceberg/changelog.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
-
-.. include:: ../../providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst
diff --git a/docs/conf.py b/docs/conf.py
index bb6440d2331..288a007fe7a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -828,6 +828,9 @@ if PACKAGE_NAME.startswith("apache-airflow-providers-"):
)
)
+ # Here we remove all other providers from the autoapi list, only leaving
the current provider,
+ # Otherwise all the other provider indexes will no be found in any TOC.
+
for p in load_package_data(include_suspended=True):
if p["package-name"] == PACKAGE_NAME:
continue
@@ -840,6 +843,9 @@ if PACKAGE_NAME.startswith("apache-airflow-providers-"):
autoapi_dirs.append(test_dir)
autoapi_ignore.extend(f"{d}/*" for d in test_dir.glob("*") if
d.is_dir() and d.name != "system")
+ print("#### AUTOAPI_IGNORE:")
+ print(autoapi_ignore)
+ print("#### END OF AUTOAPI_IGNORE:")
else:
if SYSTEM_TESTS_DIR and os.path.exists(SYSTEM_TESTS_DIR):
autoapi_dirs.append(SYSTEM_TESTS_DIR)
diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py
index 91f9f510528..7ef60d4ddb0 100644
--- a/docs/exts/provider_yaml_utils.py
+++ b/docs/exts/provider_yaml_utils.py
@@ -129,15 +129,13 @@ def load_package_data(include_suspended: bool = False) ->
list[dict[str, Any]]:
raise RuntimeError(msg)
if provider["state"] == "suspended" and not include_suspended:
continue
- package_module = provider["package-name"].replace("-", ".")
- package_module_path = package_module.replace(".", "/")
- provider_yaml_dir_str = os.path.dirname(new_provider_yaml_path.parent
/ "src" / package_module_path)
- provider["python-module"] = provider["package-name"].replace("-", ".")
- provider["package-dir"] = provider_yaml_dir_str
+ provider_yaml_dir_str = os.path.dirname(new_provider_yaml_path)
+ module = provider["package-name"][len("apache-") :].replace("-", ".")
+ module_folder = module[len("airflow-providers-") :].replace(".", "/")
+ provider["python-module"] = module
+ provider["package-dir"] =
f"{provider_yaml_dir_str}/src/{module.replace('.', '/')}"
provider["docs-dir"] = os.path.dirname(new_provider_yaml_path.parent /
"docs")
- provider["system-tests-dir"] = (
- (Path(provider_yaml_dir_str) / "tests" /
"system").relative_to(AIRFLOW_PROVIDERS_DIR).as_posix()
- )
+ provider["system-tests-dir"] =
f"{provider_yaml_dir_str}/tests/system/{module_folder}"
# TODO(potiuk) - remove when all providers are new-style
provider["is_new_provider"] = True
result.append(provider)
diff --git a/providers/airbyte/docs/index.rst b/providers/airbyte/docs/index.rst
index 0904b37a100..2037587a70d 100644
--- a/providers/airbyte/docs/index.rst
+++ b/providers/airbyte/docs/index.rst
@@ -48,14 +48,14 @@
:maxdepth: 1
:caption: System tests
- System Tests <_api/tests/system/providers/airbyte/index>
+ System Tests <_api/tests/system/airbyte/index>
.. toctree::
:hidden:
:maxdepth: 1
:caption: Resources
- Example DAGs
<https://github.com/apache/airflow/tree/providers-airbyte/|version|/providers/tests/system/airbyte>
+ Example DAGs
<https://github.com/apache/airflow/tree/providers-airbyte/|version|/providers/airbyte/tests/system/airbyte>
PyPI Repository
<https://pypi.org/project/apache-airflow-providers-airbyte/>
Installing from sources <installing-providers-from-sources>
diff --git a/providers/airbyte/docs/operators/airbyte.rst
b/providers/airbyte/docs/operators/airbyte.rst
index 8ae40766e6d..78ee38606e5 100644
--- a/providers/airbyte/docs/operators/airbyte.rst
+++ b/providers/airbyte/docs/operators/airbyte.rst
@@ -47,14 +47,14 @@ This Operator will initiate the Airbyte job, and the
Operator manages the job st
An example using the synchronous way:
-.. exampleinclude::
/../../providers/airbyte/tests/system/providers/airbyte/example_airbyte_trigger_job.py
+.. exampleinclude::
/../../providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py
:language: python
:start-after: [START howto_operator_airbyte_synchronous]
:end-before: [END howto_operator_airbyte_synchronous]
An example using the async way:
-.. exampleinclude::
/../../providers/airbyte/tests/system/providers/airbyte/example_airbyte_trigger_job.py
+.. exampleinclude::
/../../providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py
:language: python
:start-after: [START howto_operator_airbyte_asynchronous]
:end-before: [END howto_operator_airbyte_asynchronous]
diff --git a/providers/airbyte/provider.yaml b/providers/airbyte/provider.yaml
index 27ae7337b96..5b64d283ff4 100644
--- a/providers/airbyte/provider.yaml
+++ b/providers/airbyte/provider.yaml
@@ -53,7 +53,7 @@ versions:
integrations:
- integration-name: Airbyte
external-doc-url: https://airbyte.com/
- logo: /docs/integration-logos/Airbyte.png
+ logo: /docs/integration-logo/Airbyte.png
how-to-guide:
- /docs/apache-airflow-providers-airbyte/operators/airbyte.rst
tags: [service]
diff --git
a/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py
b/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py
index 58f86594414..b88fabe63ea 100644
--- a/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py
+++ b/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py
@@ -58,7 +58,7 @@ def get_provider_info():
{
"integration-name": "Airbyte",
"external-doc-url": "https://airbyte.com/";,
- "logo": "/docs/integration-logos/Airbyte.png",
+ "logo": "/docs/integration-logo/Airbyte.png",
"how-to-guide":
["/docs/apache-airflow-providers-airbyte/operators/airbyte.rst"],
"tags": ["service"],
}
diff --git a/providers/airbyte/tests/system/providers/airbyte/__init__.py
b/providers/airbyte/tests/system/airbyte/__init__.py
similarity index 100%
rename from providers/airbyte/tests/system/providers/airbyte/__init__.py
rename to providers/airbyte/tests/system/airbyte/__init__.py
diff --git
a/providers/airbyte/tests/system/providers/airbyte/example_airbyte_trigger_job.py
b/providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py
similarity index 100%
rename from
providers/airbyte/tests/system/providers/airbyte/example_airbyte_trigger_job.py
rename to providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py
diff --git a/providers/apache/iceberg/README.rst
b/providers/apache/iceberg/README.rst
new file mode 100644
index 00000000000..3ac613d01ac
--- /dev/null
+++ b/providers/apache/iceberg/README.rst
@@ -0,0 +1,61 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE
TEMPLATE
+ `PROVIDER_README_TEMPLATE.rst.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-apache-iceberg``
+
+Release: ``1.2.0``
+
+
+`Iceberg <https://iceberg.apache.org/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``apache.iceberg`` provider. All classes for
this provider package
+are in ``airflow.providers.apache.iceberg`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation
<https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/1.2.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see
``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-apache-iceberg``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+================== ==================
+PIP package Version required
+================== ==================
+``apache-airflow`` ``>=2.9.0``
+================== ==================
+
+The changelog for the provider package can be found in the
+`changelog
<https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/1.2.0/changelog.html>`_.
diff --git
a/providers/src/airflow/providers/apache/iceberg/.latest-doc-only-change.txt
b/providers/apache/iceberg/docs/.latest-doc-only-change.txt
similarity index 100%
rename from
providers/src/airflow/providers/apache/iceberg/.latest-doc-only-change.txt
rename to providers/apache/iceberg/docs/.latest-doc-only-change.txt
diff --git a/providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst
b/providers/apache/iceberg/docs/changelog.rst
similarity index 100%
rename from providers/src/airflow/providers/apache/iceberg/CHANGELOG.rst
rename to providers/apache/iceberg/docs/changelog.rst
diff --git a/docs/apache-airflow-providers-apache-iceberg/commits.rst
b/providers/apache/iceberg/docs/commits.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-iceberg/commits.rst
rename to providers/apache/iceberg/docs/commits.rst
diff --git a/docs/apache-airflow-providers-apache-iceberg/connections.rst
b/providers/apache/iceberg/docs/connections.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-iceberg/connections.rst
rename to providers/apache/iceberg/docs/connections.rst
diff --git a/docs/apache-airflow-providers-apache-iceberg/index.rst
b/providers/apache/iceberg/docs/index.rst
similarity index 97%
rename from docs/apache-airflow-providers-apache-iceberg/index.rst
rename to providers/apache/iceberg/docs/index.rst
index 9f9c915705a..5cfeef892cb 100644
--- a/docs/apache-airflow-providers-apache-iceberg/index.rst
+++ b/providers/apache/iceberg/docs/index.rst
@@ -49,7 +49,7 @@
:maxdepth: 1
:caption: Resources
- Example DAGs
<https://github.com/apache/airflow/tree/providers-apache-iceberg/|version|/providers/tests/system/apache/iceberg>
+ Example DAGs
<https://github.com/apache/airflow/tree/providers-apache-iceberg/|version|/providers/apache/iceberg/tests/system/apache/iceberg>
PyPI Repository
<https://pypi.org/project/apache-airflow-providers-apache-iceberg/>
Installing from sources <installing-providers-from-sources>
Python API <_api/airflow/providers/apache/iceberg/index>
diff --git
a/docs/apache-airflow-providers-apache-iceberg/installing-providers-from-sources.rst
b/providers/apache/iceberg/docs/installing-providers-from-sources.rst
similarity index 100%
rename from
docs/apache-airflow-providers-apache-iceberg/installing-providers-from-sources.rst
rename to providers/apache/iceberg/docs/installing-providers-from-sources.rst
diff --git a/docs/integration-logos/iceberg/iceberg.png
b/providers/apache/iceberg/docs/integration-logo/iceberg.png
similarity index 100%
rename from docs/integration-logos/iceberg/iceberg.png
rename to providers/apache/iceberg/docs/integration-logo/iceberg.png
diff --git a/docs/apache-airflow-providers-apache-iceberg/security.rst
b/providers/apache/iceberg/docs/security.rst
similarity index 100%
rename from docs/apache-airflow-providers-apache-iceberg/security.rst
rename to providers/apache/iceberg/docs/security.rst
diff --git a/providers/src/airflow/providers/apache/iceberg/provider.yaml
b/providers/apache/iceberg/provider.yaml
similarity index 91%
rename from providers/src/airflow/providers/apache/iceberg/provider.yaml
rename to providers/apache/iceberg/provider.yaml
index 14840d6487e..6f7c69aa8ab 100644
--- a/providers/src/airflow/providers/apache/iceberg/provider.yaml
+++ b/providers/apache/iceberg/provider.yaml
@@ -29,15 +29,9 @@ versions:
- 1.1.0
- 1.0.0
-dependencies:
- - apache-airflow>=2.9.0
-
-devel-dependencies:
- - pyiceberg>=0.5.0
-
integrations:
- integration-name: Iceberg
- logo: /integration-logos/iceberg/iceberg.png
+ logo: /docs/integration-logo/iceberg.png
external-doc-url: https://iceberg.apache.org/
tags: [software]
diff --git
a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
b/providers/apache/iceberg/pyproject.toml
similarity index 56%
copy from dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
copy to providers/apache/iceberg/pyproject.toml
index b3a6ad8871b..cfc1cf1023d 100644
--- a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
+++ b/providers/apache/iceberg/pyproject.toml
@@ -1,22 +1,4 @@
-{#
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-#}
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -43,9 +25,9 @@ requires = ["flit_core==3.10.1"]
build-backend = "flit_core.buildapi"
[project]
-name = "{{ PACKAGE_PIP_NAME }}"
-version = "{{RELEASE}}{{ VERSION_SUFFIX }}"
-description = "Provider package {{ PACKAGE_PIP_NAME }} for Apache Airflow"
+name = "apache-airflow-providers-apache-iceberg"
+version = "1.2.0"
+description = "Provider package apache-airflow-providers-apache-iceberg for
Apache Airflow"
readme = "README.rst"
authors = [
{name="Apache Software Foundation", email="[email protected]"},
@@ -53,7 +35,7 @@ authors = [
maintainers = [
{name="Apache Software Foundation", email="[email protected]"},
]
-keywords = [ "airflow-provider", "{{ PROVIDER_ID }}", "airflow", "integration"
]
+keywords = [ "airflow-provider", "apache.iceberg", "airflow", "integration" ]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
@@ -63,28 +45,28 @@ classifiers = [
"Framework :: Apache Airflow",
"Framework :: Apache Airflow :: Provider",
"License :: OSI Approved :: Apache Software License",
- {%- for python_version in SUPPORTED_PYTHON_VERSIONS %}
- "Programming Language :: Python :: {{ python_version }}",
- {%- endfor %}
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
"Topic :: System :: Monitoring",
]
requires-python = "~=3.9"
# The dependencies should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
dependencies = [
-{{ INSTALL_REQUIREMENTS }}
+ "apache-airflow>=2.9.0",
]
-
-{%- if EXTRAS_REQUIREMENTS %}
-# The optional dependencies should be modified in place in the generated file
+# The dependency groups should be modified in place in the generated file
# Any change in the dependencies is preserved when the file is regenerated
-[project.optional-dependencies]
-{{ EXTRAS_REQUIREMENTS }}
-{%- endif %}
+[dependency-groups]
+dev = [
+ "pyiceberg>=0.5.0"
+]
[project.urls]
-"Documentation" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME
}}/{{RELEASE}}"
-"Changelog" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME
}}/{{RELEASE}}/changelog.html"
+"Documentation" =
"https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/1.2.0";
+"Changelog" =
"https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/1.2.0/changelog.html";
"Bug Tracker" = "https://github.com/apache/airflow/issues";
"Source Code" = "https://github.com/apache/airflow";
"Slack Chat" = "https://s.apache.org/airflow-slack";
@@ -92,17 +74,10 @@ dependencies = [
"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/";
[project.entry-points."apache_airflow_provider"]
-provider_info = "airflow.providers.{{ PROVIDER_ID
}}.get_provider_info:get_provider_info"
-
-{%- if PLUGINS %}
-[project.entry-points."airflow.plugins"]
-{%- for plugin in PLUGINS %}
-{{ plugin.name }} = "{{ plugin.package_name }}:{{ plugin.class_name }}"
-{%- endfor %}
-{%- endif %}
+provider_info =
"airflow.providers.apache.iceberg.get_provider_info:get_provider_info"
[tool.flit.module]
-name = "airflow.providers.{{ PROVIDER_ID }}"
+name = "airflow.providers.apache.iceberg"
[tool.pytest.ini_options]
ignore = "tests/system/"
diff --git
a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/LICENSE
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/LICENSE
new file mode 100644
index 00000000000..11069edd790
--- /dev/null
+++ b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/providers/src/airflow/providers/apache/iceberg/__init__.py
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py
similarity index 100%
rename from providers/src/airflow/providers/apache/iceberg/__init__.py
rename to
providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py
diff --git
a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py
new file mode 100644
index 00000000000..c26da6aba33
--- /dev/null
+++
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+#
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `get_provider_info_TEMPLATE.py.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+def get_provider_info():
+ return {
+ "package-name": "apache-airflow-providers-apache-iceberg",
+ "name": "Apache Iceberg",
+ "description": "`Iceberg <https://iceberg.apache.org/>`__\n",
+ "state": "ready",
+ "source-date-epoch": 1734528119,
+ "versions": ["1.2.0", "1.1.0", "1.0.0"],
+ "integrations": [
+ {
+ "integration-name": "Iceberg",
+ "logo": "/docs/integration-logo/iceberg.png",
+ "external-doc-url": "https://iceberg.apache.org/";,
+ "tags": ["software"],
+ }
+ ],
+ "hooks": [
+ {
+ "integration-name": "Iceberg",
+ "python-modules":
["airflow.providers.apache.iceberg.hooks.iceberg"],
+ }
+ ],
+ "connection-types": [
+ {
+ "hook-class-name":
"airflow.providers.apache.iceberg.hooks.iceberg.IcebergHook",
+ "connection-type": "iceberg",
+ }
+ ],
+ "dependencies": ["apache-airflow>=2.9.0"],
+ "devel-dependencies": ["pyiceberg>=0.5.0"],
+ }
diff --git a/providers/src/airflow/providers/apache/iceberg/hooks/__init__.py
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/hooks/__init__.py
similarity index 100%
rename from providers/src/airflow/providers/apache/iceberg/hooks/__init__.py
rename to
providers/apache/iceberg/src/airflow/providers/apache/iceberg/hooks/__init__.py
diff --git a/providers/src/airflow/providers/apache/iceberg/hooks/iceberg.py
b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/hooks/iceberg.py
similarity index 100%
rename from providers/src/airflow/providers/apache/iceberg/hooks/iceberg.py
rename to
providers/apache/iceberg/src/airflow/providers/apache/iceberg/hooks/iceberg.py
diff --git a/providers/tests/apache/iceberg/hooks/__init__.py
b/providers/apache/iceberg/tests/conftest.py
similarity index 58%
copy from providers/tests/apache/iceberg/hooks/__init__.py
copy to providers/apache/iceberg/tests/conftest.py
index 13a83393a91..068fe6bbf5a 100644
--- a/providers/tests/apache/iceberg/hooks/__init__.py
+++ b/providers/apache/iceberg/tests/conftest.py
@@ -14,3 +14,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
+import pathlib
+
+import pytest
+
+pytest_plugins = "tests_common.pytest_plugin"
+
+
[email protected](tryfirst=True)
+def pytest_configure(config: pytest.Config) -> None:
+ deprecations_ignore_path =
pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml")
+ dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists()
else []
+ config.inicfg["airflow_deprecations_ignore"] = (
+ config.inicfg.get("airflow_deprecations_ignore", []) + dep_path #
type: ignore[assignment,operator]
+ )
diff --git a/providers/tests/apache/iceberg/hooks/__init__.py
b/providers/apache/iceberg/tests/providers/__init__.py
similarity index 87%
copy from providers/tests/apache/iceberg/hooks/__init__.py
copy to providers/apache/iceberg/tests/providers/__init__.py
index 13a83393a91..31d3edfae03 100644
--- a/providers/tests/apache/iceberg/hooks/__init__.py
+++ b/providers/apache/iceberg/tests/providers/__init__.py
@@ -14,3 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type:
ignore
diff --git a/providers/tests/apache/iceberg/hooks/__init__.py
b/providers/apache/iceberg/tests/providers/apache/__init__.py
similarity index 87%
copy from providers/tests/apache/iceberg/hooks/__init__.py
copy to providers/apache/iceberg/tests/providers/apache/__init__.py
index 13a83393a91..31d3edfae03 100644
--- a/providers/tests/apache/iceberg/hooks/__init__.py
+++ b/providers/apache/iceberg/tests/providers/apache/__init__.py
@@ -14,3 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type:
ignore
diff --git a/providers/tests/apache/iceberg/__init__.py
b/providers/apache/iceberg/tests/providers/apache/iceberg/__init__.py
similarity index 100%
rename from providers/tests/apache/iceberg/__init__.py
rename to providers/apache/iceberg/tests/providers/apache/iceberg/__init__.py
diff --git a/providers/tests/apache/iceberg/hooks/__init__.py
b/providers/apache/iceberg/tests/providers/apache/iceberg/hooks/__init__.py
similarity index 100%
rename from providers/tests/apache/iceberg/hooks/__init__.py
rename to
providers/apache/iceberg/tests/providers/apache/iceberg/hooks/__init__.py
diff --git a/providers/tests/apache/iceberg/hooks/test_iceberg.py
b/providers/apache/iceberg/tests/providers/apache/iceberg/hooks/test_iceberg.py
similarity index 100%
rename from providers/tests/apache/iceberg/hooks/test_iceberg.py
rename to
providers/apache/iceberg/tests/providers/apache/iceberg/hooks/test_iceberg.py
diff --git a/providers/tests/system/apache/iceberg/__init__.py
b/providers/apache/iceberg/tests/system/apache/iceberg/__init__.py
similarity index 100%
rename from providers/tests/system/apache/iceberg/__init__.py
rename to providers/apache/iceberg/tests/system/apache/iceberg/__init__.py
diff --git a/providers/tests/system/apache/iceberg/example_iceberg.py
b/providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py
similarity index 100%
rename from providers/tests/system/apache/iceberg/example_iceberg.py
rename to
providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py
diff --git a/pyproject.toml b/pyproject.toml
index 4907a91463e..2b5a6b5cf61 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -607,6 +607,7 @@ dev = [
"local-providers",
"apache-airflow-task-sdk",
"apache-airflow-providers-airbyte",
+ "apache-airflow-providers-apache-iceberg",
"apache-airflow-providers-celery",
"apache-airflow-providers-edge",
]
@@ -617,8 +618,16 @@ dev = [
local-providers = { workspace = true }
apache-airflow-task-sdk = { workspace = true }
apache-airflow-providers-airbyte = {workspace = true}
+apache-airflow-providers-apache-iceberg = {workspace = true}
apache-airflow-providers-celery = {workspace = true}
apache-airflow-providers-edge = {workspace = true}
[tool.uv.workspace]
-members = ["providers", "task_sdk", "providers/airbyte", "providers/celery",
"providers/edge" ]
+members = [
+ "providers",
+ "task_sdk",
+ "providers/airbyte",
+ "providers/apache/iceberg",
+ "providers/celery",
+ "providers/edge"
+]
diff --git a/scripts/ci/pre_commit/update_providers_dependencies.py
b/scripts/ci/pre_commit/update_providers_dependencies.py
index f5c9c018abb..6f4a6792623 100755
--- a/scripts/ci/pre_commit/update_providers_dependencies.py
+++ b/scripts/ci/pre_commit/update_providers_dependencies.py
@@ -238,9 +238,14 @@ if __name__ == "__main__":
ALL_DEPENDENCIES[provider]["deps"].extend(
PYPROJECT_TOML_CONTENT[provider]["project"]["dependencies"]
)
+ dependency_groups =
PYPROJECT_TOML_CONTENT[provider].get("dependency-groups")
+ if dependency_groups and dependency_groups.get("dev"):
+
ALL_DEPENDENCIES[provider]["devel-deps"].extend(dependency_groups["dev"])
else:
ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"])
-
ALL_DEPENDENCIES[provider]["devel-deps"].extend(provider_yaml_content.get("devel-dependencies")
or [])
+ ALL_DEPENDENCIES[provider]["devel-deps"].extend(
+ provider_yaml_content.get("devel-dependencies") or []
+ )
ALL_DEPENDENCIES[provider]["plugins"].extend(provider_yaml_content.get("plugins")
or [])
STATES[provider] = provider_yaml_content["state"]
if warnings:
