This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-10-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit d973bc6856 Update default branches for 2-10
add ed290e7928 Removing changelog:skip validation for helm chart issue
generation (#40934)
add 12a3ab3701 Fix compatibility for older airflow versions for tests
(#40938)
add 193605a308 Add Rom as committer (#40937)
add 5b289330d0 Feature/add token authentication to internal api (#40899)
add f1852c2ab2 Validating provider documentation urls before displaying in
views (#40933)
add 7f20b1eed8 Extend get_datasets endpoint to include dataset aliases
(#40830)
add e9d2d5c86a Retrieve inlet dataset events through dataset aliases
(#40809)
add 88298603eb Fix log endpoint query broken in #40791 (#40950)
add 8a912f9fa0 [AIP-62] Translate AIP-60 URI to OpenLineage (#40173)
add 9ec9eb79a0 openlineage: Add AirflowRunFacet for dag runEvents (#40854)
add 0206a4cbcf openlineage: migrate OpenLineage provider to V2 facets.
(#39530)
add 84d19bc0d0 Add more combination of SBOM files and fixes generation for
new CDXGEN (#40959)
add efca50afe0 Improve ATTRIBUTE_REMOVED sentinel to use class and more
context (#40920)
add 579a8b87fc openlineage: extend custom_run_facets to also be executed
on complete and fail (#40953)
add b713b302a6 Update styles for renderedjson component (#40964)
add 6adae0b425 openlineage: add support for hook lineage for Object Store
(#40829)
add c2304d2d30 Remove Kubernetes 1.26 support (#40943)
add 460d2095cd Format DAG run count (#39684)
add 7d35e040d3 Update metrics names to allow multiple executors to report
metrics (#40778)
add 2161e0ac2d Run full tests when tests/utils change. (#40965)
add 9596148620 AIP-64: Document Task Instance History (#40969)
add 965d752443 Small fixes to verification procedure for python client
(#40977)
add 8e72c8ca28 Upgrade UV to 0.2.28 (#40978)
add 29f456a85f Bugfix for get_parsing_context() when ran with
LocalExecutor (#40738)
add 0eaa6117bf Move sensitive information to the secret manager for the
system test google_analytics_admin (#40951)
add 2a6d3471cd test(serialization): add test case
test_dag_deps_datasets_with_duplicate_dataset to verify that duplicate datasets
in schedule and outlet of a DAG is deduplicated (#40984)
add 170f28d783 fix escape variable AIRFLOW_UID with 3087089 in
docker-compose.yaml command (#40986)
add eca05550d3 Set `slots` to True for facets used in DagRun (#40972)
add 72da823cb8 Update default branches for 2-10
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d973bc6856)
\
N -- N -- N refs/heads/v2-10-stable (72da823cb8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 -
.pre-commit-config.yaml | 14 +-
Dockerfile | 2 +-
Dockerfile.ci | 4 +-
README.md | 16 +-
airflow/api_connexion/endpoints/log_endpoint.py | 31 ++-
airflow/api_connexion/schemas/dataset_schema.py | 14 ++
airflow/api_internal/endpoints/rpc_api_endpoint.py | 47 +++-
airflow/api_internal/internal_api_call.py | 13 +-
airflow/api_internal/openapi/internal_api_v1.yaml | 1 -
airflow/config_templates/config.yml | 13 ++
airflow/datasets/__init__.py | 30 ++-
airflow/executors/base_executor.py | 72 ++++--
airflow/executors/local_executor.py | 11 +-
airflow/io/path.py | 48 +++-
airflow/lineage/hook.py | 12 +-
airflow/models/baseoperator.py | 8 +-
airflow/models/taskinstance.py | 10 +-
airflow/provider.yaml.schema.json | 4 +
airflow/providers/amazon/aws/datasets/s3.py | 22 ++
airflow/providers/amazon/aws/operators/athena.py | 33 ++-
airflow/providers/amazon/aws/operators/s3.py | 18 +-
.../providers/amazon/aws/operators/sagemaker.py | 5 +-
airflow/providers/amazon/provider.yaml | 4 +-
airflow/providers/common/compat/__init__.py | 2 +-
.../common/compat/openlineage}/__init__.py | 0
.../providers/common/compat/openlineage/facet.py | 158 +++++++++++++
airflow/providers/common/io/datasets/file.py | 26 +++
.../providers/common/io/operators/file_transfer.py | 3 +-
airflow/providers/common/io/provider.yaml | 3 +-
airflow/providers/ftp/operators/ftp.py | 3 +-
.../providers/google/cloud/openlineage/mixins.py | 104 ++++++---
.../providers/google/cloud/openlineage/utils.py | 36 +--
airflow/providers/google/cloud/operators/gcs.py | 15 +-
.../google/cloud/transfers/azure_blob_to_gcs.py | 3 +-
.../google/cloud/transfers/bigquery_to_gcs.py | 13 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 13 +-
.../providers/google/cloud/transfers/gcs_to_gcs.py | 3 +-
.../operators/analytics_admin.py | 1 +
airflow/providers/google/provider.yaml | 1 +
airflow/providers/openlineage/extractors/base.py | 25 ++-
airflow/providers/openlineage/extractors/bash.py | 6 +-
.../providers/openlineage/extractors/manager.py | 34 +--
airflow/providers/openlineage/extractors/python.py | 6 +-
.../openlineage/facets/AirflowDagRunFacet.json | 105 +++++++++
airflow/providers/openlineage/plugins/adapter.py | 110 +++++----
airflow/providers/openlineage/plugins/facets.py | 24 +-
airflow/providers/openlineage/plugins/listener.py | 12 +-
airflow/providers/openlineage/sqlparser.py | 30 +--
airflow/providers/openlineage/utils/sql.py | 10 +-
airflow/providers/openlineage/utils/utils.py | 77 +++++--
airflow/providers/sftp/operators/sftp.py | 3 +-
airflow/providers/snowflake/hooks/snowflake.py | 3 +-
airflow/providers/snowflake/provider.yaml | 1 +
.../snowflake/transfers/copy_into_snowflake.py | 13 +-
airflow/providers_manager.py | 82 ++++---
airflow/serialization/serialized_objects.py | 4 +-
airflow/utils/context.py | 40 +++-
airflow/utils/docs.py | 12 +-
airflow/utils/types.py | 9 +-
.../www/static/js/components/RenderedJsonField.tsx | 23 +-
airflow/www/static/js/dags.js | 10 +-
chart/files/statsd-mappings.yml | 15 ++
contributing-docs/08_static_code_checks.rst | 2 +
dev/README_RELEASE_AIRFLOW.md | 6 +-
dev/README_RELEASE_PYTHON_CLIENT.md | 6 +-
dev/breeze/doc/images/output-commands.svg | 42 ++--
.../doc/images/output_k8s_configure-cluster.svg | 6 +-
.../doc/images/output_k8s_configure-cluster.txt | 2 +-
.../doc/images/output_k8s_create-cluster.svg | 6 +-
.../doc/images/output_k8s_create-cluster.txt | 2 +-
.../doc/images/output_k8s_delete-cluster.svg | 4 +-
.../doc/images/output_k8s_delete-cluster.txt | 2 +-
.../doc/images/output_k8s_deploy-airflow.svg | 6 +-
.../doc/images/output_k8s_deploy-airflow.txt | 2 +-
dev/breeze/doc/images/output_k8s_k9s.svg | 4 +-
dev/breeze/doc/images/output_k8s_k9s.txt | 2 +-
dev/breeze/doc/images/output_k8s_logs.svg | 4 +-
dev/breeze/doc/images/output_k8s_logs.txt | 2 +-
.../doc/images/output_k8s_run-complete-tests.svg | 6 +-
.../doc/images/output_k8s_run-complete-tests.txt | 2 +-
dev/breeze/doc/images/output_k8s_shell.svg | 4 +-
dev/breeze/doc/images/output_k8s_shell.txt | 2 +-
dev/breeze/doc/images/output_k8s_status.svg | 4 +-
dev/breeze/doc/images/output_k8s_status.txt | 2 +-
dev/breeze/doc/images/output_k8s_tests.svg | 6 +-
dev/breeze/doc/images/output_k8s_tests.txt | 2 +-
.../doc/images/output_k8s_upload-k8s-image.svg | 6 +-
.../doc/images/output_k8s_upload-k8s-image.txt | 2 +-
.../images/output_sbom_update-sbom-information.svg | 80 ++++---
.../images/output_sbom_update-sbom-information.txt | 2 +-
dev/breeze/doc/images/output_static-checks.svg | 28 +--
dev/breeze/doc/images/output_static-checks.txt | 2 +-
.../commands/release_management_commands.py | 2 +-
.../src/airflow_breeze/commands/sbom_commands.py | 149 ++++++++++---
.../commands/sbom_commands_config.py | 3 +
dev/breeze/src/airflow_breeze/global_constants.py | 5 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
dev/breeze/src/airflow_breeze/utils/cdxgen.py | 96 ++++++--
.../src/airflow_breeze/utils/selective_checks.py | 15 +-
dev/breeze/tests/test_packages.py | 3 +
dev/breeze/tests/test_selective_checks.py | 78 ++++---
.../guides/developer.rst | 89 ++++----
.../logging-monitoring/metrics.rst | 3 +
.../authoring-and-scheduling/datasets.rst | 24 ++
docs/apache-airflow/core-concepts/dag-run.rst | 17 +-
.../howto/docker-compose/docker-compose.yaml | 4 +-
docs/apache-airflow/img/task_instance_history.png | Bin 0 -> 118727 bytes
.../img/task_instance_history_log.png | Bin 0 -> 156398 bytes
docs/apache-airflow/project.rst | 1 +
generated/PYPI_README.md | 16 +-
generated/provider_dependencies.json | 8 +
hatch_build.py | 1 +
helm_tests/other/test_statsd.py | 5 +-
...=> check_common_compat_used_for_openlineage.py} | 14 +-
...py => check_get_lineage_collector_providers.py} | 18 +-
.../ci/pre_commit/check_tests_in_right_folders.py | 1 +
.../endpoints/test_dataset_endpoint.py | 7 +-
tests/api_connexion/schemas/test_dataset_schema.py | 14 +-
.../endpoints/test_rpc_api_endpoint.py | 80 +++++--
tests/api_internal/test_internal_api_call.py | 44 ++--
tests/core/test_configuration.py | 1 +
tests/dags/test_openlineage_execution.py | 3 +-
tests/datasets/test_dataset.py | 48 +++-
tests/executors/test_base_executor.py | 48 +++-
tests/io/test_path.py | 43 +++-
tests/io/test_wrapper.py | 64 ++++++
tests/models/test_taskinstance.py | 146 ++++++++++++
tests/providers/amazon/aws/datasets/test_s3.py | 54 ++++-
.../providers/amazon/aws/operators/test_athena.py | 36 +--
.../amazon/aws/operators/test_redshift_sql.py | 42 ++--
tests/providers/amazon/aws/operators/test_s3.py | 18 +-
.../aws/operators/test_sagemaker_processing.py | 2 +-
.../aws/operators/test_sagemaker_training.py | 2 +-
.../aws/operators/test_sagemaker_transform.py | 2 +-
.../common/compat/openlineage}/__init__.py | 0
.../compat/openlineage/test_facet.py} | 4 +-
tests/providers/common/io/datasets/test_file.py | 45 +++-
.../common/io/operators/test_file_transfer.py | 3 +-
.../common/sql/operators/test_sql_execute.py | 16 +-
.../providers/dbt/cloud/utils/test_openlineage.py | 19 +-
tests/providers/ftp/operators/test_ftp.py | 2 +-
.../google/cloud/openlineage/test_mixins.py | 66 +++---
.../google/cloud/openlineage/test_utils.py | 28 +--
.../google/cloud/operators/test_bigquery.py | 12 +-
tests/providers/google/cloud/operators/test_gcs.py | 19 +-
.../cloud/transfers/test_azure_blob_to_gcs.py | 2 +-
.../google/cloud/transfers/test_bigquery_to_gcs.py | 34 +--
.../google/cloud/transfers/test_gcs_to_bigquery.py | 84 ++++---
.../google/cloud/transfers/test_gcs_to_gcs.py | 8 +-
tests/providers/mysql/operators/test_mysql.py | 16 +-
.../providers/openlineage/extractors/test_base.py | 21 +-
.../providers/openlineage/extractors/test_bash.py | 4 +-
.../openlineage/extractors/test_manager.py | 44 ++--
.../openlineage/extractors/test_python.py | 4 +-
.../providers/openlineage/plugins/test_adapter.py | 247 +++++++++++++--------
tests/providers/openlineage/plugins/test_facets.py | 85 ++++++-
.../providers/openlineage/plugins/test_listener.py | 41 +++-
tests/providers/openlineage/test_sqlparser.py | 46 ++--
.../openlineage/utils/custom_facet_fixture.py | 59 ++---
tests/providers/openlineage/utils/test_sql.py | 17 +-
tests/providers/openlineage/utils/test_utils.py | 146 +++++++-----
tests/providers/sftp/operators/test_sftp.py | 2 +-
.../snowflake/operators/test_snowflake_sql.py | 19 +-
.../transfers/test_copy_into_snowflake.py | 20 +-
tests/providers/trino/operators/test_trino.py | 23 +-
tests/serialization/test_dag_serialization.py | 65 ++++++
tests/serialization/test_pydantic_models.py | 6 +-
.../marketing_platform/example_analytics_admin.py | 43 +++-
tests/test_utils/db.py | 8 +-
tests/www/views/test_views.py | 31 +++
tests/www/views/test_views_dataset.py | 2 +-
172 files changed, 2979 insertions(+), 1236 deletions(-)
copy airflow/{api_connexion =>
providers/common/compat/openlineage}/__init__.py (100%)
create mode 100644 airflow/providers/common/compat/openlineage/facet.py
create mode 100644 airflow/providers/openlineage/facets/AirflowDagRunFacet.json
create mode 100644 docs/apache-airflow/img/task_instance_history.png
create mode 100644 docs/apache-airflow/img/task_instance_history_log.png
copy scripts/ci/pre_commit/{check_cncf_k8s_used_for_k8s_executor_only.py =>
check_common_compat_used_for_openlineage.py} (81%)
copy scripts/ci/pre_commit/{check_airflow_k8s_not_used.py =>
check_get_lineage_collector_providers.py} (78%)
create mode 100644 tests/io/test_wrapper.py
copy {airflow/api_connexion =>
tests/providers/common/compat/openlineage}/__init__.py (100%)
copy tests/providers/{tabular/hooks/test_tabular.py =>
common/compat/openlineage/test_facet.py} (86%)
