This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch openlineage-systemtests
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 651b265f3cb add basic system tests for OpenLineage
add 6e59137781b Fix bug in task_sdk's `parse` function (#44056)
add f66459b5694 Migrate public endpoint Get Tasks to FastAPI (#43980)
add 817f0803a8d Remove console log from TriggerDag (#44050)
add feb6f7f3025 Remove commented code in xcom_arg.py (#44058)
add e1b4f073f41 Fix flakey test in TaskSDK supervisor (#44064)
add 0f14f66994a AIP-84: Migrating GET Dataset events for DAG runs api to
fastAPI (#43874)
add fd0a0d2eb2f AIP 84: Migrate POST ASSET EVENT legacy API to fast API.
(#43984)
add bafa9da6fc1 AIP-72: Use explicit TI State enums for TI update state
endpoint (#44065)
add da9c4017d19 Remove pre-commit from devel extra of Airflow (#44059)
add eed59f1e96f Exclude `src/airflow/__init__.py` when building providers
(#44066)
add aa72f0f3312 AIP-84 Get Batch Task Instances (#44051)
add 4dfae23532d Update DAG example links in multiple providers documents
(#44034)
add 5c442d378dc Modularize datamodels in Execution API (#44068)
add 98bda4c5c06 Log message source details are grouped (#43681)
add cf7b6a43e24 Fix system test `test_aws_auth_manager` (#44073)
add f7270c8a202 Extend OpenAPI schema with extra models for Task SDK
(#44076)
add 39177301874 AIP-84: Migrating GET queued asset events for DAG to
fastAPI (#43934)
add d8bff0094ee make host/port configurable for Snowflake connections
(#44079)
add bf124d0b2bf Fix special tests in the new workflow structure (#44083)
add a41feeb5aed Re-queue tassk when they are stuck in queued (#43520)
add fa2e4e953f9 Bump cross-spawn from 7.0.3 to 7.0.5 in /airflow/www
(#44085)
add 2db0d11c156 Refactor SerializedDagModel and DagCode for dag versioning
(#43821)
add d8ebda4f4b1 Optimize test execution time after refactors (#44089)
add b5f564663fe Revert "AIP-84: Migrating GET queued asset events for DAG
to fastAPI (#43934)" (#44088)
add 16bc3826c38 fix test_handle_stuck_queued_tasks_multiple_attempts
(#44093)
add 83883fae10a Ignore requests RetryError in k8s tests (#44094)
add 30c97d66579 Fix error propagation in new tests (#44087)
add 8c53a20c6fe Do not reset DB in CI tests if not needed (#44084)
add 5d6b836c612 Add .dockerignore to target workflow override (#43885)
add fb758ae56bc Migrate Edge Worker backend to FastAPI (#43865)
add 79bb80b4b6c Drop support for PostgreSQL 12 (#44080)
add 3256e432d71 Remove deprecated functionality from Google provider
(#43953)
add f5d92a8a165 Add newsfragment PR 43393 (#44091)
add 80a2f106404 Fix command quoting in breeze README (#44090)
add a85d94e6cdc Add usedforsecurity for sha1 algorithm (#44081)
add 696c799ddd9 Fix duration charts (#44075)
add 76ce15a4c32 Fix Pydantic model in Airflow 2.10 back-compat tests for
Edge (#44099)
add f0dcfd65bd5 Correct type hinting for RPC API endpoint in EdgeWorker
for FastAPI (#44097)
add 295eea2b505 Add cherry picker (#44102)
add b6c75d2cd6b update standard provider CHANGELOG.rst (#44110)
add 987c50a0ae9 BUG Update `initialise_virtualenv.py` to use `uv sync`
(#44096)
add 36803b8bd9a Move filesystem sensor to standard provider (#43890)
add 8d8bb0f23e1 fix docker documentation auth url (#44112)
add c807762ec65 Move cherry pick config to root and better rename for
backport workflow (#44116)
add abc97edd0aa Bump cross-spawn from 7.0.3 to 7.0.5 in /airflow/ui
(#44117)
add 15a577cd2bd Add cherry-picker documentation (#44118)
add 2cc9b5865ba Fix missing end of block in Airflow 3 DEV readme (#44120)
add 5085151db0f Repair internal API on main for Edge Worker / RPC API
(#44098)
add eb1153e36dc Bump to mypy-boto3-appflow and pass without `# type:
ignore[arg-type]` (#44115)
add 7c11944be71 Feat: telegram send file Operator and Hook (#44040)
add 4863383dca5 Update Edge Executor documentation to current state
(#44119)
add 681651d7e7c Add option --version-suffix-for-local (#43769)
add 3f43bc4a5f4 partial kwargs deserialized MappedOperator set on unmapped
Operator (#42563)
add 7b8bd2823a1 AIP-84: Migrating GET queued asset events for DAG to
fastAPI (#44124)
add 6f02fdbe2e9 get_task_instance_try_details API returns
TaskInstanceHistory schema (#43830)
add 007c4b16595 Improve dag_maker compatibility handling (#44125)
add 32efbd033fe Update providers metadata 2024-11-18 (#44132)
add f3bad1e9e17 AIP-84 Clean status code (#44134)
add 6c3caa6f311 AIP-84: Migrating DELETE queued asset events for DAG to
fastAPI (#44129)
add 802705581cd AIP-38 | List Import Errors (#44021)
add 313be641ed2 Redact extra fields for Asset Endpoints in fastAPI (#44069)
add f834a637216 doc: providers github link (#44136)
add f860ec9742b Docs: Improve PostgreSQL db and user setup for clarity
(#44067)
add 23f141d5b2a AIP-72: Add "XCom" POST endpoint for Execution API (#44101)
add 869d9965414 Init Dag Overview page with time range selector and failed
tasks (#44074)
add 298f1fd73b5 Resolve circular import issue related to task sdk (#44086)
add 1acb54e6879 Fix link to events page (#44154)
add 49daa6c25f4 AIP-84 Allow only positive limit and offset (#44109)
add 6faa720cd8f Remove XCom pickling (#43905)
add 2bd701a68a1 Improve IDE type hinting of methdtools.lru_cache in Task
SDK client (#44152)
add 29a30b3e62a AIP-84: Migrating DELETE queued asset events for assets to
FASTAPI (#44138)
add 347a83afbd5 Exclude Scarf Usage Data Collection in CI Environments
(#44155)
add e9b59fc2592 Ensure that the Task SDK regularly sends heartbeats for
running tasks (#44162)
add 4bce3a353ac Bugfix KubernetesJobOperator.on_kill() by removing
unexpected `job` argument from the k8s job client method
`delete_namespaced_job` invocation (#44131)
add f5abe506fc6 fix(providers/databricks): remove additional argument
passed to repair_run (#44140)
add 9eaeb1c3098 suppress the warnings where we check for sensitive values
(#44148)
add b968307acf0 Remove commented breakpoint in dbt provider (#44163)
add 1bcd7ec2a4f fix typo in scheduler_job_runner (#44165)
add af958bbf923 AIP-84: Migrating GET one queued asset events for DAG to
fastAPI (#44128)
add 8f9631ece1c AIP-84: Migrating DELETE a queued asset events for DAG to
fastAPI (#44130)
add a60d105c103 Use `S3CopyObjectOperator` in
`example_comprehend_document_classifier` (#44160)
add 55b30b28eae use status (#44161)
add 275e48a062a Lower bound spy calls check (#44177)
add 21fc21d2600 Add task sdk tests checks to slack notifier (#44174)
add cc8aa7bc42e AIP-65: Update dag source endpoint to support versioning
(#43492)
add 672056f2a69 spark-submit: replace `principle` by `principal` (#44150)
add 8daf74e606a Replace `principle` by `principal` in kerberos-related
code (#43679)
add 5f7f95bfb65 Migrate magic numbers for status codes to named constants
(#44176)
add c83768a05e2 AIP-84 constraint wildcard path params for get batch TI
(#44175)
add e964c640cf2 AIP-84 Get Task Instance Try Details (#43675)
add 87337da1017 fix get ti endpoint route (#44171)
add 24811f729f0 Make task output "unbuffered" so output is captured
straight away (#44186)
add e8fe1bddf1b raise 404 when dag or dag_run_id is not found in List TI
endpoint (#44156)
add 39042c8c031 AIP-84: Migrating GET queued asset events for assets to
FASTAPI (#44139)
add 86c4c6fa9de Migrate pickled data & change XCom value type to JSON
(#44166)
new 3a97b276318 add basic system tests for OpenLineage
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (651b265f3cb)
\
N -- N -- N refs/heads/openlineage-systemtests (3a97b276318)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../__init__.py => .cherry_picker.toml | 7 +
.github/actions/checkout_target_commit/action.yml | 7 +-
.github/actions/install-pre-commit/action.yml | 11 +-
.github/workflows/automatic-backport.yml | 77 +
.github/workflows/backport-cli.yml | 115 ++
.github/workflows/ci.yml | 2 +
.github/workflows/special-tests.yml | 30 +-
Dockerfile | 22 +-
Dockerfile.ci | 28 +-
README.md | 2 +-
RELEASE_NOTES.rst | 4 +-
airflow/api/common/delete_dag.py | 6 -
airflow/api_connexion/endpoints/asset_endpoint.py | 7 +
.../api_connexion/endpoints/dag_run_endpoint.py | 1 +
.../api_connexion/endpoints/dag_source_endpoint.py | 58 +-
airflow/api_connexion/endpoints/task_endpoint.py | 1 +
.../endpoints/task_instance_endpoint.py | 2 +
airflow/api_connexion/endpoints/xcom_endpoint.py | 2 +-
airflow/api_connexion/openapi/v1.yaml | 112 +-
airflow/api_connexion/schemas/dag_source_schema.py | 2 +
airflow/api_fastapi/common/parameters.py | 108 +-
airflow/api_fastapi/common/types.py | 28 -
airflow/api_fastapi/core_api/datamodels/assets.py | 48 +-
.../api_fastapi/core_api/datamodels/dag_sources.py | 2 +
.../core_api/datamodels/task_instances.py | 69 +-
airflow/api_fastapi/core_api/datamodels/tasks.py | 47 +-
.../api_fastapi/core_api/openapi/v1-generated.yaml | 992 ++++++++++++-
.../api_fastapi/core_api/routes/public/assets.py | 265 +++-
.../core_api/routes/public/connections.py | 28 +-
.../api_fastapi/core_api/routes/public/dag_run.py | 36 +-
.../core_api/routes/public/dag_sources.py | 25 +-
.../core_api/routes/public/event_logs.py | 1 -
.../core_api/routes/public/import_error.py | 5 +-
.../api_fastapi/core_api/routes/public/pools.py | 4 +-
.../core_api/routes/public/task_instances.py | 173 ++-
.../api_fastapi/core_api/routes/public/tasks.py | 39 +-
.../core_api/routes/public/variables.py | 2 +-
airflow/api_fastapi/core_api/routes/public/xcom.py | 2 +-
airflow/api_fastapi/execution_api/app.py | 46 +-
.../execution_api/datamodels}/__init__.py | 0
.../execution_api/datamodels/connection.py} | 19 +-
.../{datamodels.py => datamodels/taskinstance.py} | 80 +-
.../api_fastapi/execution_api/datamodels/token.py | 13 +-
.../execution_api/datamodels/variable.py} | 13 +-
.../api_fastapi/execution_api/datamodels/xcom.py | 13 +-
airflow/api_fastapi/execution_api/deps.py | 8 +-
.../execution_api/routes/connections.py | 10 +-
.../execution_api/routes/task_instances.py | 15 +-
.../api_fastapi/execution_api/routes/variables.py | 13 +-
airflow/api_fastapi/execution_api/routes/xcoms.py | 92 +-
airflow/api_internal/endpoints/rpc_api_endpoint.py | 6 +-
airflow/config_templates/config.yml | 9 -
airflow/configuration.py | 3 +-
airflow/dag_processing/manager.py | 6 +-
airflow/decorators/base.py | 2 +-
airflow/example_dags/example_sensors.py | 2 +-
airflow/executors/base_executor.py | 27 +-
airflow/jobs/scheduler_job_runner.py | 169 ++-
.../versions/0047_3_0_0_add_dag_versioning.py | 9 +-
...49_3_0_0_remove_pickled_data_from_xcom_table.py | 182 +++
airflow/models/abstractoperator.py | 5 +-
airflow/models/baseoperator.py | 9 +-
airflow/models/dag.py | 3 +-
airflow/models/dag_version.py | 25 +-
airflow/models/dagcode.py | 48 +-
airflow/models/mappedoperator.py | 2 +
airflow/models/param.py | 2 +-
airflow/models/serialized_dag.py | 82 +-
airflow/models/taskinstance.py | 38 +-
airflow/models/variable.py | 2 +-
airflow/models/xcom.py | 36 +-
airflow/models/xcom_arg.py | 4 +-
airflow/reproducible_build.yaml | 4 +-
airflow/security/kerberos.py | 4 +-
airflow/sensors/base.py | 7 +-
airflow/serialization/pydantic/dag_run.py | 3 +-
airflow/serialization/pydantic/taskinstance.py | 2 +
airflow/serialization/schema.json | 1 -
airflow/serialization/serialized_objects.py | 4 +-
airflow/template/templater.py | 2 +-
airflow/ui/openapi-gen/queries/common.ts | 158 +-
airflow/ui/openapi-gen/queries/prefetch.ts | 188 ++-
airflow/ui/openapi-gen/queries/queries.ts | 444 +++++-
airflow/ui/openapi-gen/queries/suspense.ts | 224 ++-
airflow/ui/openapi-gen/requests/schemas.gen.ts | 544 ++++++-
airflow/ui/openapi-gen/requests/services.gen.ts | 372 ++++-
airflow/ui/openapi-gen/requests/types.gen.ts | 483 ++++++-
airflow/ui/package.json | 6 +-
airflow/ui/pnpm-lock.yaml | 782 +---------
airflow/ui/src/components/StateCircle.tsx | 2 +-
airflow/ui/src/components/TimeRangeSelector.tsx | 21 +-
.../ui/src/components/TriggerDag/TriggerDag.tsx | 1 -
airflow/ui/src/layouts/Nav/BrowseButton.tsx | 6 +-
airflow/ui/src/pages/DagsList/Dag/Code/Code.tsx | 10 +-
.../ui/src/pages/DagsList/Dag/Overview/Chart.tsx | 160 ++
.../src/pages/DagsList/Dag/Overview/Overview.tsx | 96 ++
.../Dag/Overview/index.ts} | 16 +-
airflow/ui/src/pages/DagsList/DagCard.tsx | 6 +-
airflow/ui/src/pages/Dashboard/Dashboard.tsx | 4 +
.../ui/src/pages/Dashboard/{ => Health}/Health.tsx | 6 +-
.../pages/Dashboard/{ => Health}/HealthSection.tsx | 0
.../src/pages/Dashboard/{ => Health}/HealthTag.tsx | 0
.../Dashboard/{Dashboard.tsx => Health/index.ts} | 16 +-
.../HistoricalMetrics/HistoricalMetrics.tsx | 13 +-
.../src/pages/Dashboard/Stats/DAGImportErrors.tsx | 78 +
.../pages/Dashboard/Stats/DAGImportErrorsModal.tsx | 125 ++
.../Dashboard/{Dashboard.tsx => Stats/Stats.tsx} | 22 +-
.../Dashboard/{Dashboard.tsx => Stats/index.ts} | 16 +-
airflow/ui/src/router.tsx | 3 +-
airflow/utils/db.py | 2 +-
airflow/utils/edgemodifier.py | 2 +-
airflow/utils/file.py | 2 +-
airflow/utils/log/file_task_handler.py | 6 +-
airflow/utils/state.py | 58 +-
airflow/utils/usage_data_collection.py | 25 +
airflow/www/static/js/api/useDagCode.ts | 8 +-
.../js/cluster-activity/live-metrics/Pools.tsx | 2 +-
.../static/js/dag/details/dag/RunDurationChart.tsx | 2 +-
.../static/js/dag/details/task/AllTaskDuration.tsx | 2 +-
.../static/js/dag/details/task/TaskDuration.tsx | 2 +-
airflow/www/static/js/types/api-generated.ts | 92 +-
airflow/www/templates/airflow/dag.html | 2 +-
airflow/www/views.py | 2 +-
airflow/www/yarn.lock | 16 +-
chart/newsfragments/43393.improvement.rst | 1 +
contributing-docs/03_contributors_quick_start.rst | 2 +-
contributing-docs/11_provider_packages.rst | 53 +-
dev/README_AIRFLOW3_DEV.md | 84 ++
dev/backport/update_backport_status.py | 91 ++
dev/breeze/README.md | 2 +-
dev/breeze/doc/ci/02_images.md | 2 +-
dev/breeze/doc/images/output-commands.svg | 16 +-
...elease-management_prepare-provider-packages.svg | 52 +-
...elease-management_prepare-provider-packages.txt | 2 +-
dev/breeze/doc/images/output_setup_config.svg | 2 +-
dev/breeze/doc/images/output_setup_config.txt | 2 +-
dev/breeze/doc/images/output_shell.svg | 16 +-
dev/breeze/doc/images/output_shell.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 16 +-
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
.../output_testing_core-integration-tests.svg | 40 +-
.../output_testing_core-integration-tests.txt | 2 +-
.../doc/images/output_testing_core-tests.svg | 48 +-
.../doc/images/output_testing_core-tests.txt | 2 +-
.../output_testing_providers-integration-tests.svg | 40 +-
.../output_testing_providers-integration-tests.txt | 2 +-
.../doc/images/output_testing_providers-tests.svg | 48 +-
.../doc/images/output_testing_providers-tests.txt | 2 +-
.../doc/images/output_testing_system-tests.svg | 40 +-
.../doc/images/output_testing_system-tests.txt | 2 +-
.../src/airflow_breeze/commands/common_options.py | 4 +-
.../commands/release_management_commands.py | 23 +-
.../commands/release_management_commands_config.py | 1 +
.../airflow_breeze/commands/testing_commands.py | 8 +-
dev/breeze/src/airflow_breeze/global_constants.py | 6 +-
.../src/airflow_breeze/params/shell_params.py | 7 +-
.../prepare_providers/provider_packages.py | 6 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 34 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 86 +-
.../src/airflow_breeze/utils/selective_checks.py | 6 +-
.../src/airflow_breeze/utils/version_utils.py | 54 +
dev/breeze/tests/test_selective_checks.py | 12 +-
dev/images/backport_commit_action.png | Bin 0 -> 295983 bytes
docs/apache-airflow-providers-airbyte/index.rst | 2 +-
docs/apache-airflow-providers-alibaba/index.rst | 2 +-
.../example-dags.rst | 2 +-
.../apache-airflow-providers-apache-beam/index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../apache-airflow-providers-apache-hive/index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../apache-airflow-providers-apache-livy/index.rst | 2 +-
docs/apache-airflow-providers-apache-pig/index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
docs/apache-airflow-providers-asana/index.rst | 2 +-
docs/apache-airflow-providers-common-io/index.rst | 2 +-
docs/apache-airflow-providers-common-sql/index.rst | 2 +-
docs/apache-airflow-providers-databricks/index.rst | 2 +-
docs/apache-airflow-providers-dingding/index.rst | 2 +-
.../connections/docker.rst | 2 +-
docs/apache-airflow-providers-docker/index.rst | 2 +-
.../edge_executor.rst | 13 +-
.../index.rst | 2 +-
docs/apache-airflow-providers-ftp/index.rst | 2 +-
docs/apache-airflow-providers-github/index.rst | 2 +-
.../connections/gcp.rst | 7 +-
.../operators/cloud/automl.rst | 2 +-
.../operators/cloud/dataflow.rst | 8 +-
.../operators/cloud/gcs.rst | 14 -
.../operators/cloud/vertex_ai.rst | 13 +-
.../operators/marketing_platform/analytics.rst | 53 -
docs/apache-airflow-providers-http/index.rst | 2 +-
docs/apache-airflow-providers-influxdb/index.rst | 2 +-
docs/apache-airflow-providers-jdbc/index.rst | 2 +-
docs/apache-airflow-providers-jenkins/index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
.../index.rst | 2 +-
docs/apache-airflow-providers-mysql/index.rst | 2 +-
docs/apache-airflow-providers-neo4j/index.rst | 2 +-
docs/apache-airflow-providers-opsgenie/index.rst | 2 +-
docs/apache-airflow-providers-papermill/index.rst | 2 +-
docs/apache-airflow-providers-postgres/index.rst | 2 +-
docs/apache-airflow-providers-presto/index.rst | 2 +-
docs/apache-airflow-providers-redis/index.rst | 2 +-
docs/apache-airflow-providers-salesforce/index.rst | 2 +-
.../apache-airflow-providers-singularity/index.rst | 2 +-
docs/apache-airflow-providers-slack/index.rst | 2 +-
.../connections/snowflake.rst | 2 +
docs/apache-airflow-providers-snowflake/index.rst | 2 +-
docs/apache-airflow-providers-sqlite/index.rst | 2 +-
.../sensors}/file.rst | 2 +-
docs/apache-airflow-providers-tableau/index.rst | 2 +-
docs/apache-airflow-providers-telegram/index.rst | 2 +-
docs/apache-airflow-providers-trino/index.rst | 2 +-
docs/apache-airflow-providers-yandex/index.rst | 2 +-
docs/apache-airflow-providers-ydb/index.rst | 2 +-
docs/apache-airflow-providers-zendesk/index.rst | 2 +-
docs/apache-airflow-providers/index.rst | 2 +-
docs/apache-airflow/howto/define-extra-link.rst | 32 +-
docs/apache-airflow/howto/operator/index.rst | 1 -
docs/apache-airflow/howto/set-up-database.rst | 3 +
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 329 +++--
docs/apache-airflow/migrations-ref.rst | 4 +-
docs/apache-airflow/operators-and-hooks-ref.rst | 3 -
docs/apache-airflow/tutorial/taskflow.rst | 2 +-
docs/spelling_wordlist.txt | 1 +
generated/PYPI_README.md | 2 +-
generated/provider_dependencies.json | 2 +-
generated/provider_metadata.json | 270 +++-
hatch_build.py | 1 -
kubernetes_tests/test_base.py | 5 +-
newsfragments/43679.misc.rst | 1 +
newsfragments/44080.significant.rst | 3 +
newsfragments/aip-72.significant.rst | 14 +
providers/pyproject.toml | 1 +
.../airflow/providers/amazon/aws/hooks/appflow.py | 19 +-
.../src/airflow/providers/amazon/aws/hooks/s3.py | 5 +
.../airflow/providers/amazon/aws/operators/s3.py | 5 +
.../src/airflow/providers/amazon/provider.yaml | 4 +-
.../providers/apache/spark/hooks/spark_submit.py | 10 +-
.../providers/celery/executors/celery_executor.py | 44 +-
.../celery/executors/celery_kubernetes_executor.py | 29 +-
.../kubernetes/executors/kubernetes_executor.py | 53 +-
.../executors/kubernetes_executor_types.py | 10 +
.../executors/kubernetes_executor_utils.py | 29 +-
.../executors/local_kubernetes_executor.py | 12 +
.../providers/cncf/kubernetes/operators/job.py | 1 -
.../airflow/providers/common/io/xcom/backend.py | 22 +-
.../providers/databricks/operators/databricks.py | 2 +-
.../src/airflow/providers/dbt/cloud/hooks/dbt.py | 1 -
providers/src/airflow/providers/edge/CHANGELOG.rst | 8 +
providers/src/airflow/providers/edge/__init__.py | 2 +-
.../edge/api_endpoints/rpc_api_endpoint.py | 150 --
.../src/airflow/providers/edge/openapi/__init__.py | 3 +
.../providers/edge/openapi/edge_worker_api_v1.yaml | 6 +-
.../providers/edge/plugins/edge_executor_plugin.py | 36 +-
providers/src/airflow/providers/edge/provider.yaml | 2 +-
.../airflow/providers/edge/worker_api}/__init__.py | 2 +-
.../src/airflow/providers/edge/worker_api/app.py | 39 +
.../providers/edge/worker_api/datamodels.py | 17 +-
.../providers/edge/worker_api/routes}/__init__.py | 0
.../providers/edge/worker_api/routes/_v2_compat.py | 102 ++
.../providers/edge/worker_api/routes/health.py | 14 +-
.../providers/edge/worker_api/routes/rpc_api.py | 219 +++
.../src/airflow/providers/google/CHANGELOG.rst | 146 ++
.../src/airflow/providers/google/ads/hooks/ads.py | 3 +-
.../airflow/providers/google/cloud/hooks/automl.py | 6 +-
.../providers/google/cloud/hooks/bigquery.py | 1522 +-------------------
.../providers/google/cloud/hooks/bigquery_dts.py | 12 +-
.../providers/google/cloud/hooks/bigtable.py | 6 +-
.../providers/google/cloud/hooks/cloud_batch.py | 6 +-
.../providers/google/cloud/hooks/cloud_build.py | 15 +-
.../providers/google/cloud/hooks/cloud_composer.py | 16 -
.../google/cloud/hooks/cloud_memorystore.py | 8 +-
.../providers/google/cloud/hooks/cloud_run.py | 12 +-
.../providers/google/cloud/hooks/cloud_sql.py | 9 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 7 +-
.../providers/google/cloud/hooks/compute.py | 6 +-
.../providers/google/cloud/hooks/compute_ssh.py | 5 -
.../providers/google/cloud/hooks/datacatalog.py | 6 +-
.../providers/google/cloud/hooks/dataflow.py | 14 +-
.../providers/google/cloud/hooks/dataform.py | 8 -
.../providers/google/cloud/hooks/datafusion.py | 14 +-
.../providers/google/cloud/hooks/dataplex.py | 8 +-
.../providers/google/cloud/hooks/dataprep.py | 4 +-
.../providers/google/cloud/hooks/dataproc.py | 20 +-
.../google/cloud/hooks/dataproc_metastore.py | 8 -
.../providers/google/cloud/hooks/datastore.py | 6 +-
.../airflow/providers/google/cloud/hooks/dlp.py | 6 +-
.../providers/google/cloud/hooks/functions.py | 6 +-
.../airflow/providers/google/cloud/hooks/gcs.py | 17 -
.../airflow/providers/google/cloud/hooks/gdm.py | 18 +-
.../airflow/providers/google/cloud/hooks/kms.py | 6 +-
.../google/cloud/hooks/kubernetes_engine.py | 101 +-
.../providers/google/cloud/hooks/life_sciences.py | 6 +-
.../airflow/providers/google/cloud/hooks/looker.py | 3 +-
.../providers/google/cloud/hooks/mlengine.py | 8 -
.../google/cloud/hooks/natural_language.py | 6 +-
.../providers/google/cloud/hooks/os_login.py | 6 +-
.../airflow/providers/google/cloud/hooks/pubsub.py | 6 +-
.../providers/google/cloud/hooks/secret_manager.py | 73 +-
.../providers/google/cloud/hooks/spanner.py | 6 +-
.../providers/google/cloud/hooks/speech_to_text.py | 6 +-
.../providers/google/cloud/hooks/stackdriver.py | 6 +-
.../airflow/providers/google/cloud/hooks/tasks.py | 6 +-
.../providers/google/cloud/hooks/text_to_speech.py | 6 +-
.../providers/google/cloud/hooks/translate.py | 6 +-
.../google/cloud/hooks/vertex_ai/auto_ml.py | 6 +-
.../cloud/hooks/vertex_ai/batch_prediction_job.py | 6 +-
.../google/cloud/hooks/vertex_ai/custom_job.py | 6 +-
.../google/cloud/hooks/vertex_ai/dataset.py | 8 -
.../cloud/hooks/vertex_ai/endpoint_service.py | 8 -
.../cloud/hooks/vertex_ai/generative_model.py | 15 +-
.../hooks/vertex_ai/hyperparameter_tuning_job.py | 6 +-
.../google/cloud/hooks/vertex_ai/model_service.py | 8 -
.../google/cloud/hooks/vertex_ai/pipeline_job.py | 1 +
.../google/cloud/hooks/video_intelligence.py | 6 +-
.../airflow/providers/google/cloud/hooks/vision.py | 6 +-
.../providers/google/cloud/hooks/workflows.py | 8 -
.../providers/google/cloud/links/dataproc.py | 1 -
.../providers/google/cloud/operators/automl.py | 4 +-
.../providers/google/cloud/operators/bigquery.py | 342 +----
.../providers/google/cloud/operators/dataflow.py | 516 -------
.../providers/google/cloud/operators/dataproc.py | 630 --------
.../google/cloud/operators/kubernetes_engine.py | 10 -
.../google/cloud/operators/vertex_ai/auto_ml.py | 92 --
.../operators/vertex_ai/batch_prediction_job.py | 13 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 16 -
.../vertex_ai/hyperparameter_tuning_job.py | 13 +-
.../google/cloud/secrets/secret_manager.py | 19 +-
.../providers/google/cloud/sensors/bigquery.py | 80 -
.../google/cloud/sensors/cloud_composer.py | 93 +-
.../airflow/providers/google/cloud/sensors/gcs.py | 35 +-
.../airflow/providers/google/cloud/triggers/gcs.py | 2 +-
.../providers/google/common/hooks/base_google.py | 12 +-
.../providers/google/common/hooks/discovery_api.py | 5 -
.../google/marketing_platform/hooks/analytics.py | 211 ---
.../marketing_platform/hooks/campaign_manager.py | 2 -
.../marketing_platform/hooks/display_video.py | 4 +-
.../google/marketing_platform/hooks/search_ads.py | 9 +-
.../marketing_platform/operators/analytics.py | 551 -------
.../operators/campaign_manager.py | 41 -
.../marketing_platform/operators/display_video.py | 46 -
.../marketing_platform/sensors/campaign_manager.py | 6 -
.../marketing_platform/sensors/display_video.py | 12 -
.../src/airflow/providers/google/provider.yaml | 14 -
.../providers/google/suite/hooks/calendar.py | 7 -
.../airflow/providers/google/suite/hooks/drive.py | 5 -
.../airflow/providers/google/suite/hooks/sheets.py | 6 -
.../providers/google/suite/operators/sheets.py | 6 -
.../providers/google/suite/sensors/drive.py | 6 -
.../google/suite/transfers/gcs_to_gdrive.py | 6 -
.../google/suite/transfers/gcs_to_sheets.py | 6 -
.../google/suite/transfers/local_to_drive.py | 6 -
.../google/suite/transfers/sql_to_sheets.py | 6 -
.../providers/microsoft/azure/operators/adx.py | 3 +-
.../providers/microsoft/winrm/operators/winrm.py | 3 +-
.../providers/openlineage/transport/variable.py | 10 +-
.../airflow/providers/snowflake/hooks/snowflake.py | 8 +
.../src/airflow/providers/ssh/operators/ssh.py | 3 +-
.../src/airflow/providers/standard/CHANGELOG.rst | 2 +
.../src/airflow/providers/standard/provider.yaml | 1 +
.../providers/standard}/sensors/filesystem.py | 0
.../airflow/providers/telegram/hooks/telegram.py | 30 +
.../providers/telegram/operators/telegram.py | 56 +
.../tests/amazon/aws/log/test_s3_task_handler.py | 6 +-
.../tests/amazon/aws/operators/test_base_aws.py | 7 +-
.../tests/amazon/aws/sensors/test_base_aws.py | 7 +-
.../tests/apache/spark/hooks/test_spark_submit.py | 6 +-
.../tests/celery/executors/test_celery_executor.py | 36 +-
.../tests/celery/log_handlers/test_log_handlers.py | 4 +-
.../executors/test_kubernetes_executor.py | 67 +-
.../tests/cncf/kubernetes/operators/test_job.py | 14 +-
.../kubernetes/operators/test_spark_kubernetes.py | 10 +-
providers/tests/deprecations_ignore.yml | 11 -
.../edge/api_endpoints/test_rpc_api_endpoint.py | 282 ----
.../edge/plugins/test_edge_executor_plugin.py | 11 +-
.../tests/edge/worker_api}/__init__.py | 0
.../tests/edge/worker_api/routes}/__init__.py | 0
.../routes/test_health.py} | 4 +-
.../tests/edge/worker_api/routes/test_rpc_api.py | 297 ++++
.../api_endpoints/test_dag_source_endpoint.py | 16 +-
providers/tests/google/cloud/hooks/test_automl.py | 5 -
.../tests/google/cloud/hooks/test_bigquery.py | 759 +---------
.../tests/google/cloud/hooks/test_bigquery_dts.py | 8 -
.../tests/google/cloud/hooks/test_bigtable.py | 5 -
.../tests/google/cloud/hooks/test_cloud_build.py | 8 -
.../google/cloud/hooks/test_cloud_composer.py | 8 -
.../google/cloud/hooks/test_cloud_memorystore.py | 4 -
.../tests/google/cloud/hooks/test_cloud_sql.py | 4 -
.../hooks/test_cloud_storage_transfer_service.py | 4 -
providers/tests/google/cloud/hooks/test_compute.py | 4 -
.../tests/google/cloud/hooks/test_compute_ssh.py | 4 -
.../tests/google/cloud/hooks/test_datacatalog.py | 4 -
.../tests/google/cloud/hooks/test_dataflow.py | 8 -
.../tests/google/cloud/hooks/test_dataform.py | 4 -
.../tests/google/cloud/hooks/test_datafusion.py | 8 -
.../tests/google/cloud/hooks/test_dataplex.py | 5 -
.../tests/google/cloud/hooks/test_dataproc.py | 8 -
.../google/cloud/hooks/test_dataproc_metastore.py | 4 -
.../tests/google/cloud/hooks/test_datastore.py | 4 -
providers/tests/google/cloud/hooks/test_dlp.py | 4 -
.../tests/google/cloud/hooks/test_functions.py | 4 -
providers/tests/google/cloud/hooks/test_gcs.py | 7 +-
providers/tests/google/cloud/hooks/test_gdm.py | 4 -
providers/tests/google/cloud/hooks/test_kms.py | 5 -
.../google/cloud/hooks/test_kubernetes_engine.py | 14 +-
.../tests/google/cloud/hooks/test_life_sciences.py | 4 -
.../tests/google/cloud/hooks/test_mlengine.py | 4 -
.../google/cloud/hooks/test_natural_language.py | 5 -
.../tests/google/cloud/hooks/test_os_login.py | 4 -
providers/tests/google/cloud/hooks/test_pubsub.py | 4 -
.../google/cloud/hooks/test_secret_manager.py | 57 +-
.../cloud/hooks/test_secret_manager_system.py | 34 +-
providers/tests/google/cloud/hooks/test_spanner.py | 5 -
.../google/cloud/hooks/test_speech_to_text.py | 5 -
.../tests/google/cloud/hooks/test_stackdriver.py | 4 -
providers/tests/google/cloud/hooks/test_tasks.py | 5 -
.../google/cloud/hooks/test_text_to_speech.py | 5 -
.../tests/google/cloud/hooks/test_translate.py | 6 -
.../google/cloud/hooks/test_video_intelligence.py | 5 -
providers/tests/google/cloud/hooks/test_vision.py | 4 -
.../tests/google/cloud/hooks/test_workflows.py | 6 -
.../google/cloud/hooks/vertex_ai/test_auto_ml.py | 4 -
.../hooks/vertex_ai/test_batch_prediction_job.py | 4 -
.../cloud/hooks/vertex_ai/test_custom_job.py | 4 -
.../google/cloud/hooks/vertex_ai/test_dataset.py | 4 -
.../cloud/hooks/vertex_ai/test_endpoint_service.py | 4 -
.../vertex_ai/test_hyperparameter_tuning_job.py | 4 -
.../cloud/hooks/vertex_ai/test_model_service.py | 4 -
.../google/cloud/log/test_gcs_task_handler.py | 7 +-
.../tests/google/cloud/operators/test_bigquery.py | 394 -----
.../tests/google/cloud/operators/test_dataflow.py | 361 -----
.../tests/google/cloud/operators/test_dataproc.py | 429 ------
.../tests/google/cloud/operators/test_vertex_ai.py | 191 +--
.../tests/google/cloud/sensors/test_bigquery.py | 133 +-
.../google/cloud/sensors/test_cloud_composer.py | 64 +-
providers/tests/google/cloud/sensors/test_gcs.py | 51 +-
.../cloud/triggers/test_kubernetes_engine.py | 1 -
.../tests/google/cloud/utils/base_gcp_mock.py | 4 -
.../tests/google/common/hooks/test_base_google.py | 25 -
.../marketing_platform/hooks/test_analytics.py | 191 ---
.../marketing_platform/operators/test_analytics.py | 273 ----
.../operators/test_campaign_manager.py | 5 -
.../operators/test_display_video.py | 14 -
.../sensors/test_campaign_manager.py | 1 -
.../sensors/test_display_video.py | 2 -
providers/tests/google/suite/sensors/test_drive.py | 3 -
.../google/suite/transfers/test_gcs_to_gdrive.py | 9 -
.../google/suite/transfers/test_gcs_to_sheets.py | 3 -
.../google/suite/transfers/test_sql_to_sheets.py | 1 -
.../microsoft/azure/log/test_wasb_task_handler.py | 17 +-
providers/tests/sftp/operators/test_sftp.py | 102 +-
providers/tests/standard/operators/test_bash.py | 32 +-
.../tests/standard/operators/test_datetime.py | 43 +-
providers/tests/standard/operators/test_python.py | 33 +-
providers/tests/standard/operators/test_weekday.py | 111 +-
.../aws/example_comprehend_document_classifier.py | 19 +-
.../amazon/aws/tests/test_aws_auth_manager.py | 4 +-
.../system/google/cloud/gcs/example_gcs_sensor.py | 10 -
.../vertex_ai/example_vertex_ai_custom_job.py | 2 -
.../example_vertex_ai_hyperparameter_tuning_job.py | 2 -
.../vertex_ai/example_vertex_ai_model_service.py | 2 -
.../system/openlineage/example_openlineage.py | 3 +-
.../example_openlineage_mapped_sensor.py | 5 +-
.../utils => tests/system/openlineage}/operator.py | 38 +-
providers/tests/telegram/hooks/test_telegram.py | 114 +-
scripts/ci/install_breeze.sh | 2 +-
scripts/ci/pre_commit/checkout_no_credentials.py | 7 +
.../ci/testing/run_breeze_command_with_retries.sh | 1 +
scripts/ci/testing/run_unit_tests.sh | 42 +-
scripts/docker/common.sh | 20 +-
scripts/tools/initialize_virtualenv.py | 32 +-
scripts/tools/setup_breeze | 2 +-
task_sdk/src/airflow/sdk/api/client.py | 37 +-
.../src/airflow/sdk/api/datamodels/_generated.py | 82 +-
.../src/airflow/sdk/api/datamodels/activities.py | 2 +-
task_sdk/src/airflow/sdk/definitions/dag.py | 10 +-
task_sdk/src/airflow/sdk/execution_time/comms.py | 5 +-
.../src/airflow/sdk/execution_time/supervisor.py | 17 +-
.../src/airflow/sdk/execution_time/task_runner.py | 10 +-
task_sdk/tests/conftest.py | 9 +
.../tests/dags/super_basic.py | 11 +-
task_sdk/tests/defintions/test_dag.py | 2 +-
task_sdk/tests/execution_time/test_supervisor.py | 75 +-
task_sdk/tests/execution_time/test_task_runner.py | 18 +-
tests/always/test_project_structure.py | 20 -
.../endpoints/test_dag_source_endpoint.py | 122 +-
tests/api_connexion/endpoints/test_log_endpoint.py | 23 +-
.../api_connexion/endpoints/test_xcom_endpoint.py | 32 +-
tests/api_connexion/schemas/test_xcom_schema.py | 51 -
.../core_api/routes/public/test_assets.py | 379 ++++-
.../core_api/routes/public/test_dag_run.py | 75 +
.../core_api/routes/public/test_dag_sources.py | 101 +-
.../core_api/routes/public/test_task_instances.py | 474 ++++++
.../core_api/routes/public/test_tasks.py | 254 +++-
.../core_api/routes/public/test_xcom.py | 21 -
.../api_fastapi/execution_api/routes/test_xcoms.py | 49 +-
.../api_fastapi/execution_api/test_app.py | 17 +-
.../endpoints/test_rpc_api_endpoint.py | 2 +-
tests/assets/test_manager.py | 1 +
tests/cli/commands/test_task_command.py | 2 +-
tests/jobs/test_scheduler_job.py | 133 +-
tests/models/test_dag_version.py | 37 +-
tests/models/test_dagcode.py | 8 +-
tests/models/test_serialized_dag.py | 29 +-
tests/models/test_xcom.py | 83 +-
tests/security/test_kerberos.py | 10 +-
tests/sensors/test_filesystem.py | 2 +-
tests/serialization/test_dag_serialization.py | 6 +-
tests/utils/log/test_log_reader.py | 51 +-
tests/utils/test_log_handlers.py | 8 +-
tests/utils/test_usage_data_collection.py | 2 +
tests_common/pytest_plugin.py | 127 +-
521 files changed, 10589 insertions(+), 11643 deletions(-)
rename providers/src/airflow/providers/edge/api_endpoints/__init__.py =>
.cherry_picker.toml (81%)
create mode 100644 .github/workflows/automatic-backport.yml
create mode 100644 .github/workflows/backport-cli.yml
copy airflow/{api_connexion =>
api_fastapi/execution_api/datamodels}/__init__.py (100%)
copy airflow/{api_connexion/schemas/dag_source_schema.py =>
api_fastapi/execution_api/datamodels/connection.py} (70%)
rename airflow/api_fastapi/execution_api/{datamodels.py =>
datamodels/taskinstance.py} (62%)
rename task_sdk/src/airflow/sdk/api/datamodels/ti.py =>
airflow/api_fastapi/execution_api/datamodels/token.py (83%)
copy airflow/{api_connexion/schemas/dag_source_schema.py =>
api_fastapi/execution_api/datamodels/variable.py} (75%)
copy task_sdk/src/airflow/sdk/api/datamodels/activities.py =>
airflow/api_fastapi/execution_api/datamodels/xcom.py (79%)
create mode 100644
airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py
create mode 100644 airflow/ui/src/pages/DagsList/Dag/Overview/Chart.tsx
create mode 100644 airflow/ui/src/pages/DagsList/Dag/Overview/Overview.tsx
copy airflow/ui/src/pages/{Dashboard/Dashboard.tsx =>
DagsList/Dag/Overview/index.ts} (71%)
rename airflow/ui/src/pages/Dashboard/{ => Health}/Health.tsx (94%)
rename airflow/ui/src/pages/Dashboard/{ => Health}/HealthSection.tsx (100%)
rename airflow/ui/src/pages/Dashboard/{ => Health}/HealthTag.tsx (100%)
copy airflow/ui/src/pages/Dashboard/{Dashboard.tsx => Health/index.ts} (71%)
create mode 100644 airflow/ui/src/pages/Dashboard/Stats/DAGImportErrors.tsx
create mode 100644
airflow/ui/src/pages/Dashboard/Stats/DAGImportErrorsModal.tsx
copy airflow/ui/src/pages/Dashboard/{Dashboard.tsx => Stats/Stats.tsx} (70%)
copy airflow/ui/src/pages/Dashboard/{Dashboard.tsx => Stats/index.ts} (71%)
create mode 100644 chart/newsfragments/43393.improvement.rst
create mode 100644 dev/backport/update_backport_status.py
create mode 100644 dev/images/backport_commit_action.png
delete mode 100644
docs/apache-airflow-providers-google/operators/marketing_platform/analytics.rst
rename docs/{apache-airflow/howto/operator =>
apache-airflow-providers-standard/sensors}/file.rst (92%)
create mode 100644 newsfragments/43679.misc.rst
create mode 100644 newsfragments/44080.significant.rst
delete mode 100644
providers/src/airflow/providers/edge/api_endpoints/rpc_api_endpoint.py
rename providers/{tests/edge/api_endpoints =>
src/airflow/providers/edge/worker_api}/__init__.py (93%)
create mode 100644 providers/src/airflow/providers/edge/worker_api/app.py
copy task_sdk/src/airflow/sdk/api/datamodels/activities.py =>
providers/src/airflow/providers/edge/worker_api/datamodels.py (68%)
copy {airflow/api_connexion =>
providers/src/airflow/providers/edge/worker_api/routes}/__init__.py (100%)
create mode 100644
providers/src/airflow/providers/edge/worker_api/routes/_v2_compat.py
copy airflow/api_connexion/schemas/dag_source_schema.py =>
providers/src/airflow/providers/edge/worker_api/routes/health.py (76%)
create mode 100644
providers/src/airflow/providers/edge/worker_api/routes/rpc_api.py
delete mode 100644
providers/src/airflow/providers/google/marketing_platform/hooks/analytics.py
delete mode 100644
providers/src/airflow/providers/google/marketing_platform/operators/analytics.py
rename {airflow =>
providers/src/airflow/providers/standard}/sensors/filesystem.py (100%)
delete mode 100644 providers/tests/edge/api_endpoints/test_rpc_api_endpoint.py
copy {airflow/api/auth => providers/tests/edge/worker_api}/__init__.py (100%)
copy {airflow/api/auth => providers/tests/edge/worker_api/routes}/__init__.py
(100%)
rename providers/tests/edge/{api_endpoints/test_health_endpoint.py =>
worker_api/routes/test_health.py} (88%)
create mode 100644 providers/tests/edge/worker_api/routes/test_rpc_api.py
delete mode 100644
providers/tests/google/marketing_platform/hooks/test_analytics.py
delete mode 100644
providers/tests/google/marketing_platform/operators/test_analytics.py
rename providers/{src/airflow/providers/openlineage/utils =>
tests/system/openlineage}/operator.py (89%)
rename providers/src/airflow/providers/edge/api_endpoints/health_endpoint.py
=> task_sdk/tests/dags/super_basic.py (82%)
copy airflow/api_fastapi/execution_api/deps.py =>
tests/api_fastapi/execution_api/test_app.py (60%)
