This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 2eecca17fc Change default .airflowignore syntax to glob (#42436)
add ede7cb27fd Rename dataset related python variable names to asset
(#41348)
No new revisions were added by this update.
Summary of changes:
RELEASE_NOTES.rst | 4 +-
airflow/__init__.py | 6 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 16 +-
.../api_connexion/endpoints/dataset_endpoint.py | 190 ++++---
.../schemas/{dataset_schema.py => asset_schema.py} | 88 +--
airflow/api_connexion/security.py | 8 +-
airflow/api_fastapi/openapi/v1-generated.yaml | 8 +-
airflow/api_fastapi/views/ui/__init__.py | 4 +-
.../views/ui/{datasets.py => assets.py} | 34 +-
airflow/api_internal/endpoints/rpc_api_endpoint.py | 8 +-
airflow/{datasets => assets}/__init__.py | 206 ++++----
airflow/{datasets => assets}/manager.py | 195 ++++---
airflow/{datasets => assets}/metadata.py | 10 +-
airflow/auth/managers/base_auth_manager.py | 10 +-
airflow/auth/managers/models/resource_details.py | 4 +-
.../auth/managers/simple/simple_auth_manager.py | 6 +-
airflow/config_templates/config.yml | 18 +-
airflow/dag_processing/collection.py | 147 +++---
airflow/decorators/base.py | 10 +-
airflow/example_dags/example_asset_alias.py | 101 ++++
.../example_asset_alias_with_no_taskflow.py | 108 ++++
airflow/example_dags/example_assets.py | 192 +++++++
airflow/example_dags/example_dataset_alias.py | 101 ----
.../example_dataset_alias_with_no_taskflow.py | 108 ----
airflow/example_dags/example_datasets.py | 192 -------
airflow/example_dags/example_inlet_event_extra.py | 22 +-
airflow/example_dags/example_outlet_event_extra.py | 28 +-
airflow/io/path.py | 12 +-
airflow/jobs/scheduler_job_runner.py | 90 ++--
airflow/lineage/__init__.py | 2 +-
airflow/lineage/hook.py | 126 +++--
airflow/listeners/listener.py | 4 +-
airflow/listeners/spec/{dataset.py => asset.py} | 18 +-
airflow/models/__init__.py | 2 +-
airflow/models/{dataset.py => asset.py} | 104 ++--
airflow/models/dag.py | 91 ++--
airflow/models/taskinstance.py | 74 +--
airflow/operators/python.py | 6 +-
airflow/provider.yaml.schema.json | 30 +-
.../amazon/aws/{datasets => assets}/__init__.py | 0
.../amazon/aws/{datasets => assets}/s3.py | 16 +-
.../amazon/aws/auth_manager/avp/entities.py | 2 +-
.../amazon/aws/auth_manager/aws_auth_manager.py | 16 +-
airflow/providers/amazon/aws/hooks/s3.py | 39 +-
.../aws/utils/asset_compat_lineage_collector.py | 106 ++++
airflow/providers/amazon/provider.yaml | 14 +-
airflow/providers/common/compat/assets/__init__.py | 77 +++
airflow/providers/common/compat/lineage/hook.py | 73 ++-
.../openlineage/utils}/__init__.py | 0
.../common/compat/openlineage/utils/utils.py} | 39 +-
.../common/compat/security}/__init__.py | 0
.../common/compat/security/permissions.py} | 13 +-
.../common/io/assets}/__init__.py | 0
.../common/io/{datasets => assets}/file.py | 15 +-
airflow/providers/common/io/provider.yaml | 14 +-
.../providers/fab/auth_manager/fab_auth_manager.py | 16 +-
.../fab/auth_manager/security_manager/override.py | 16 +-
airflow/providers/fab/provider.yaml | 1 +
airflow/providers/google/provider.yaml | 8 +
.../mysql/assets}/__init__.py | 0
.../providers/mysql/{datasets => assets}/mysql.py | 0
airflow/providers/mysql/provider.yaml | 8 +-
.../providers/openlineage/extractors/manager.py | 27 +-
.../utils/asset_compat_lineage_collector.py | 108 ++++
airflow/providers/openlineage/utils/utils.py | 38 +-
.../postgres/assets}/__init__.py | 0
.../postgres/{datasets => assets}/postgres.py | 0
airflow/providers/postgres/datasets/__init__.py | 16 -
airflow/providers/postgres/provider.yaml | 8 +-
.../trino/assets}/__init__.py | 0
.../providers/trino/{datasets => assets}/trino.py | 0
airflow/providers/trino/datasets/__init__.py | 16 -
airflow/providers/trino/provider.yaml | 8 +-
airflow/providers_manager.py | 48 +-
airflow/reproducible_build.yaml | 4 +-
airflow/security/permissions.py | 2 +-
airflow/serialization/dag_dependency.py | 2 +-
airflow/serialization/enums.py | 13 +-
.../pydantic/{dataset.py => asset.py} | 22 +-
airflow/serialization/pydantic/dag_run.py | 4 +-
airflow/serialization/pydantic/taskinstance.py | 4 +-
airflow/serialization/serialized_objects.py | 111 ++--
airflow/timetables/{datasets.py => assets.py} | 36 +-
airflow/timetables/base.py | 24 +-
airflow/timetables/simple.py | 40 +-
airflow/ui/openapi-gen/queries/common.ts | 18 +-
airflow/ui/openapi-gen/queries/prefetch.ts | 36 +-
airflow/ui/openapi-gen/queries/queries.ts | 21 +-
airflow/ui/openapi-gen/queries/suspense.ts | 52 +-
airflow/ui/openapi-gen/requests/services.gen.ts | 14 +-
airflow/ui/openapi-gen/requests/types.gen.ts | 6 +-
airflow/utils/context.py | 102 ++--
airflow/utils/context.pyi | 28 +-
airflow/utils/operator_helpers.py | 2 +-
airflow/www/auth.py | 6 +-
airflow/www/security_manager.py | 4 +-
airflow/www/static/css/graph.css | 8 +-
airflow/www/static/js/dag/details/graph/Node.tsx | 2 +-
airflow/www/static/js/dag/details/graph/index.tsx | 6 +-
airflow/www/static/js/dag/details/graph/utils.ts | 2 +-
airflow/www/static/js/datasets/Graph/Node.tsx | 4 +-
airflow/www/static/js/datasets/Graph/index.tsx | 2 +-
airflow/www/static/js/datasets/SearchBar.tsx | 2 +-
airflow/www/static/js/types/index.ts | 4 +-
airflow/www/templates/airflow/dag.html | 16 +-
.../www/templates/airflow/dag_dependencies.html | 4 +-
airflow/www/templates/airflow/dags.html | 22 +-
airflow/www/views.py | 102 ++--
dev/breeze/tests/test_packages.py | 3 +
.../tests/test_pytest_args_for_test_types.py | 2 +-
dev/breeze/tests/test_selective_checks.py | 26 +-
.../auth-manager/manage/index.rst | 12 +-
.../auth-manager/access-control.rst | 10 +-
.../{dataset-schemes.rst => asset-schemes.rst} | 8 +-
.../howto/create-custom-providers.rst | 4 +-
.../administration-and-deployment/lineage.rst | 12 +-
.../administration-and-deployment/listeners.rst | 8 +-
.../logging-monitoring/metrics.rst | 6 +-
.../authoring-and-scheduling/assets.rst | 532 +++++++++++++++++++
.../authoring-and-scheduling/datasets.rst | 532 -------------------
.../authoring-and-scheduling/index.rst | 2 +-
.../authoring-and-scheduling/timetable.rst | 18 +-
docs/apache-airflow/core-concepts/dag-run.rst | 4 +-
docs/apache-airflow/core-concepts/taskflow.rst | 14 +-
...scheduled-dags.png => asset-scheduled-dags.png} | Bin
.../img/{datasets.png => assets.png} | Bin
docs/apache-airflow/templates-ref.rst | 12 +-
docs/apache-airflow/tutorial/objectstorage.rst | 2 +-
docs/apache-airflow/ui.rst | 4 +-
docs/exts/operators_and_hooks_ref.py | 6 +-
...mes.rst.jinja2 => asset-uri-schemes.rst.jinja2} | 2 +-
docs/spelling_wordlist.txt | 2 +
generated/provider_dependencies.json | 10 +-
newsfragments/41348.significant.rst | 240 +++++++++
.../ci/pre_commit/check_tests_in_right_folders.py | 2 +-
scripts/cov/core_coverage.py | 2 +-
scripts/cov/other_coverage.py | 4 +-
tests/always/test_project_structure.py | 2 +
.../endpoints/test_dag_run_endpoint.py | 22 +-
.../endpoints/test_dag_source_endpoint.py | 2 +-
.../endpoints/test_dataset_endpoint.py | 170 +++---
tests/api_connexion/schemas/test_dag_schema.py | 12 +-
tests/api_connexion/schemas/test_dataset_schema.py | 90 ++--
.../views/ui/{test_datasets.py => test_assets.py} | 4 +-
.../api_connexion => tests/assets}/__init__.py | 0
tests/{datasets => assets}/test_manager.py | 104 ++--
tests/assets/tests_asset.py | 586 ++++++++++++++++++++
.../managers/simple/test_simple_auth_manager.py | 6 +-
tests/auth/managers/test_base_auth_manager.py | 6 +-
tests/conftest.py | 4 +-
tests/dags/{test_datasets.py => test_assets.py} | 6 +-
tests/dags/test_only_empty_tasks.py | 4 +-
tests/datasets/__init__.py | 16 -
tests/datasets/test_dataset.py | 588 ---------------------
tests/decorators/test_python.py | 10 +-
tests/io/test_path.py | 32 +-
tests/io/test_wrapper.py | 14 +-
tests/jobs/test_scheduler_job.py | 186 ++++---
tests/lineage/test_hook.py | 140 ++---
.../{dataset_listener.py => asset_listener.py} | 14 +-
..._dataset_listener.py => test_asset_listener.py} | 26 +-
tests/models/{test_dataset.py => test_asset.py} | 12 +-
tests/models/test_dag.py | 326 ++++++------
tests/models/test_dagrun.py | 2 +-
tests/models/test_serialized_dag.py | 14 +-
tests/models/test_taskinstance.py | 453 ++++++++--------
tests/operators/test_python.py | 4 +-
.../providers/amazon/aws/assets}/__init__.py | 0
.../amazon/aws/{datasets => assets}/test_s3.py | 22 +-
.../aws/auth_manager/test_aws_auth_manager.py | 53 +-
tests/providers/amazon/aws/datasets/__init__.py | 16 -
tests/providers/amazon/aws/hooks/test_s3.py | 50 +-
.../common/compat/openlineage/utils}/__init__.py | 0
.../common/compat/openlineage/utils/test_utils.py | 8 +-
.../providers/common/compat/security}/__init__.py | 0
.../common/compat/security/test_permissions.py | 7 +
.../providers/common/io/assets}/__init__.py | 0
.../common/io/{datasets => assets}/test_file.py | 16 +-
tests/providers/common/io/datasets/__init__.py | 16 -
.../fab/auth_manager/test_fab_auth_manager.py | 12 +-
tests/providers/fab/auth_manager/test_security.py | 14 +-
.../providers/mysql/assets}/__init__.py | 0
.../mysql/{datasets => assets}/test_mysql.py | 2 +-
tests/providers/mysql/datasets/__init__.py | 16 -
.../openlineage/extractors/test_manager.py | 35 +-
.../providers/postgres/assets}/__init__.py | 0
.../postgres/{datasets => assets}/test_postgres.py | 2 +-
tests/providers/postgres/datasets/__init__.py | 16 -
.../providers/trino/assets}/__init__.py | 0
.../trino/{datasets => assets}/test_trino.py | 2 +-
tests/providers/trino/datasets/__init__.py | 16 -
tests/serialization/test_dag_serialization.py | 86 +--
tests/serialization/test_pydantic_models.py | 52 +-
tests/serialization/test_serde.py | 6 +-
tests/serialization/test_serialized_objects.py | 48 +-
.../providers/microsoft/azure/example_msfabric.py | 4 +-
.../providers/papermill/input_notebook.ipynb | 2 +
tests/test_utils/compat.py | 33 ++
tests/test_utils/db.py | 35 +-
...asets_timetable.py => test_assets_timetable.py} | 134 +++--
tests/utils/test_context.py | 40 +-
tests/utils/test_db_cleanup.py | 6 +-
tests/utils/test_json.py | 10 +-
tests/www/test_auth.py | 2 +-
tests/www/views/test_views_acl.py | 8 +-
tests/www/views/test_views_dataset.py | 197 ++++---
tests/www/views/test_views_grid.py | 50 +-
207 files changed, 4914 insertions(+), 4286 deletions(-)
rename airflow/api_connexion/schemas/{dataset_schema.py => asset_schema.py}
(65%)
rename airflow/api_fastapi/views/ui/{datasets.py => assets.py} (66%)
rename airflow/{datasets => assets}/__init__.py (60%)
rename airflow/{datasets => assets}/manager.py (53%)
rename airflow/{datasets => assets}/metadata.py (80%)
create mode 100644 airflow/example_dags/example_asset_alias.py
create mode 100644 airflow/example_dags/example_asset_alias_with_no_taskflow.py
create mode 100644 airflow/example_dags/example_assets.py
delete mode 100644 airflow/example_dags/example_dataset_alias.py
delete mode 100644
airflow/example_dags/example_dataset_alias_with_no_taskflow.py
delete mode 100644 airflow/example_dags/example_datasets.py
copy airflow/listeners/spec/{dataset.py => asset.py} (76%)
rename airflow/models/{dataset.py => asset.py} (83%)
rename airflow/providers/amazon/aws/{datasets => assets}/__init__.py (100%)
rename airflow/providers/amazon/aws/{datasets => assets}/s3.py (73%)
create mode 100644
airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py
create mode 100644 airflow/providers/common/compat/assets/__init__.py
rename airflow/providers/common/{io/datasets =>
compat/openlineage/utils}/__init__.py (100%)
rename airflow/{listeners/spec/dataset.py =>
providers/common/compat/openlineage/utils/utils.py} (54%)
copy airflow/{api_connexion => providers/common/compat/security}/__init__.py
(100%)
copy airflow/{api_fastapi/views/ui/__init__.py =>
providers/common/compat/security/permissions.py} (71%)
copy airflow/{api_connexion => providers/common/io/assets}/__init__.py (100%)
rename airflow/providers/common/io/{datasets => assets}/file.py (76%)
copy airflow/{api_connexion => providers/mysql/assets}/__init__.py (100%)
rename airflow/providers/mysql/{datasets => assets}/mysql.py (100%)
create mode 100644
airflow/providers/openlineage/utils/asset_compat_lineage_collector.py
copy airflow/{api_connexion => providers/postgres/assets}/__init__.py (100%)
rename airflow/providers/postgres/{datasets => assets}/postgres.py (100%)
delete mode 100644 airflow/providers/postgres/datasets/__init__.py
copy airflow/{api_connexion => providers/trino/assets}/__init__.py (100%)
rename airflow/providers/trino/{datasets => assets}/trino.py (100%)
delete mode 100644 airflow/providers/trino/datasets/__init__.py
rename airflow/serialization/pydantic/{dataset.py => asset.py} (68%)
rename airflow/timetables/{datasets.py => assets.py} (71%)
rename docs/apache-airflow-providers/core-extensions/{dataset-schemes.rst =>
asset-schemes.rst} (82%)
create mode 100644 docs/apache-airflow/authoring-and-scheduling/assets.rst
delete mode 100644 docs/apache-airflow/authoring-and-scheduling/datasets.rst
rename docs/apache-airflow/img/{dataset-scheduled-dags.png =>
asset-scheduled-dags.png} (100%)
rename docs/apache-airflow/img/{datasets.png => assets.png} (100%)
rename docs/exts/templates/{dataset-uri-schemes.rst.jinja2 =>
asset-uri-schemes.rst.jinja2} (95%)
create mode 100644 newsfragments/41348.significant.rst
rename tests/api_fastapi/views/ui/{test_datasets.py => test_assets.py} (92%)
copy {airflow/api_connexion => tests/assets}/__init__.py (100%)
rename tests/{datasets => assets}/test_manager.py (52%)
create mode 100644 tests/assets/tests_asset.py
rename tests/dags/{test_datasets.py => test_assets.py} (91%)
delete mode 100644 tests/datasets/__init__.py
delete mode 100644 tests/datasets/test_dataset.py
rename tests/listeners/{dataset_listener.py => asset_listener.py} (80%)
rename tests/listeners/{test_dataset_listener.py => test_asset_listener.py}
(72%)
rename tests/models/{test_dataset.py => test_asset.py} (73%)
copy {airflow/api_connexion => tests/providers/amazon/aws/assets}/__init__.py
(100%)
rename tests/providers/amazon/aws/{datasets => assets}/test_s3.py (75%)
delete mode 100644 tests/providers/amazon/aws/datasets/__init__.py
copy {airflow/api_connexion =>
tests/providers/common/compat/openlineage/utils}/__init__.py (100%)
copy airflow/api_fastapi/views/ui/__init__.py =>
tests/providers/common/compat/openlineage/utils/test_utils.py (82%)
copy {airflow/api_connexion =>
tests/providers/common/compat/security}/__init__.py (100%)
rename airflow/providers/mysql/datasets/__init__.py =>
tests/providers/common/compat/security/test_permissions.py (81%)
copy {airflow/api_connexion => tests/providers/common/io/assets}/__init__.py
(100%)
rename tests/providers/common/io/{datasets => assets}/test_file.py (83%)
delete mode 100644 tests/providers/common/io/datasets/__init__.py
copy {airflow/api_connexion => tests/providers/mysql/assets}/__init__.py (100%)
rename tests/providers/mysql/{datasets => assets}/test_mysql.py (97%)
delete mode 100644 tests/providers/mysql/datasets/__init__.py
copy {airflow/api_connexion => tests/providers/postgres/assets}/__init__.py
(100%)
rename tests/providers/postgres/{datasets => assets}/test_postgres.py (97%)
delete mode 100644 tests/providers/postgres/datasets/__init__.py
copy {airflow/api_connexion => tests/providers/trino/assets}/__init__.py (100%)
rename tests/providers/trino/{datasets => assets}/test_trino.py (97%)
delete mode 100644 tests/providers/trino/datasets/__init__.py
rename tests/timetables/{test_datasets_timetable.py =>
test_assets_timetable.py} (57%)
