This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a change to branch handle-stuck-in-queue
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 6d639907113 Merge branch 'handle-stuck-in-queue' of
https://github.com/apache/airflow into handle-stuck-in-queue
add 028c14ad54b Fix condition for using uv in some jobs (#43630)
add b67885734f4 Full-on switching Breeze development to use uv (#43628)
add 089aa63fc4e Rebuild pgbouncer image with newer version (#42349)
add 07e6ada4bc4 Fix venv numpy example which needs to be 1.26 at least to
be working in Python 3.12 (#43653)
add c65a10f3219 Remove auth manager tests of backfill (#43649)
add 286075f0593 Add UV support to venv operators (#43612)
add 8b0ac8d5302 Pass Task Instance ids in API response (#43664)
add 3742b1ea9ee Replace HTTP status code with named constants (#43663)
add 5ccfe5c4c70 Bump `ruff` to `0.7.2` (#43668)
add 5f9fdf016da add config section to standard provider docs index (#43674)
add 90b98478cc5 Fix System test type in breeze (#43670)
add b9b06140491 AIP-72: Remove DAG pickling (#43667)
add d8f71a2d9fb Remove returns in final clause of S3ToDynamoDBOperator
(#43456)
add 73f2eab6808 serialize asset/dataset timetable conditions in
OpenLineage info also for Airflow 2 (#43434)
add 6548d50841f AIP-84 Get Event Logs (#43407)
add 18ea01cef2b Fix reproducibility of prepared provider packages (fix
flit frontend) (#43683)
add e9bc557947f Update providers metadata 2024-11-05 (#43677)
add 9ede38adad6 Change default value of `namespace` in `task.kubernetes`
to be None (#43402)
add 3939d13224a AIP-72: Add "update TI state" endpoint for Execution API
(#43602)
add da502427b72 Bump `uv` to `0.4.30` (#43692)
add 1116f286eec AIP-65: Add DAG versioning support (#42913)
add ccd65867387 Detect situation where Breeze is installed with both pipx
and uv (#43694)
add 58ceb323916 AIP-84 Get Import Error / Get Import Errors (#43637)
add c96b618b60e Disable XCom list ordering by execution_date (#43680)
add 1328d1a0340 AIP-84 Migrate post a connection to FastAPI API (#43396)
add 69c1f92eae2 Migrate the public endpoint Get DAG Stats to FastAPI
(#43255)
add f2dd3502887 Resolve `GlueJobTrigger` serialization bug causing verbose
to always be True (#43622)
add d536ec4bd1d fab_auth_manager: allow get_user method to return the user
authenticated via Kerberos (#43662)
add 74fc28020a7 GlueJobOperator: add option to wait for cleanup before
returning job status (#43688)
add 6a5b22df0e2 Replaced bootstrap-typeahead with jquery-ui (#40744)
add 438f71df8d8 Delete the Serialized Dag and DagCode before DagVersion
migration (#43700)
add ed3accb3008 Handle FileNotFound Error returned by missing uv or pipx
(#43714)
add 9f6042c2fb7 Add data interval and next_dagrun_create_after info
through tooltip to next dagrun. (#43501)
add 426dba01c08 Add `container_name` and update `awslogs_stream_prefix`
pattern (#43138)
add 5fdf2ec0ee6 Merge branch 'main' of https://github.com/apache/airflow
into handle-stuck-in-queue
No new revisions were added by this update.
Summary of changes:
.github/actions/install-pre-commit/action.yml | 2 +-
.github/workflows/build-images.yml | 4 +-
.github/workflows/ci.yml | 8 +-
.github/workflows/finalize-tests.yml | 2 +-
.github/workflows/k8s-tests.yml | 2 +-
.pre-commit-config.yaml | 4 +-
Dockerfile | 2 +-
Dockerfile.ci | 2 +-
airflow/api/common/trigger_dag.py | 5 +-
.../api_connexion/endpoints/connection_endpoint.py | 1 +
.../api_connexion/endpoints/dag_run_endpoint.py | 5 +-
.../api_connexion/endpoints/dag_stats_endpoint.py | 2 +
.../api_connexion/endpoints/event_log_endpoint.py | 1 +
.../endpoints/import_error_endpoint.py | 3 +
airflow/api_connexion/openapi/v1.yaml | 9 -
airflow/api_connexion/schemas/dag_schema.py | 1 -
airflow/api_fastapi/common/db/common.py | 5 +-
.../api_fastapi/common/db/dag_runs.py | 20 +-
airflow/api_fastapi/common/parameters.py | 19 +
.../api_fastapi/common/types.py | 8 +-
.../api_fastapi/core_api/openapi/v1-generated.yaml | 548 ++-
.../api_fastapi/core_api/routes/public/__init__.py | 4 +
.../core_api/routes/public/backfills.py | 60 +-
.../core_api/routes/public/connections.py | 45 +-
.../api_fastapi/core_api/routes/public/dag_run.py | 51 +-
.../core_api/routes/public/dag_sources.py | 16 +-
.../core_api/routes/public/dag_stats.py | 79 +
.../core_api/routes/public/dag_warning.py | 7 +-
airflow/api_fastapi/core_api/routes/public/dags.py | 80 +-
.../core_api/routes/public/event_logs.py | 103 +-
.../core_api/routes/public/import_error.py | 105 +
.../api_fastapi/core_api/routes/public/pools.py | 54 +-
.../core_api/routes/public/task_instances.py | 22 +-
.../core_api/routes/public/variables.py | 49 +-
airflow/api_fastapi/core_api/routes/ui/assets.py | 6 +-
.../api_fastapi/core_api/routes/ui/dashboard.py | 2 +-
.../core_api/serializers/connections.py | 25 +
.../serializers/{dag_warning.py => dag_stats.py} | 25 +-
airflow/api_fastapi/core_api/serializers/dags.py | 2 -
.../api_fastapi/core_api/serializers/event_logs.py | 7 +
.../serializers/{event_logs.py => import_error.py} | 27 +-
.../core_api/serializers/task_instances.py | 1 +
.../core_api/serializers/{ => ui}/dashboard.py | 0
airflow/api_fastapi/execution_api/app.py | 1 +
.../api_fastapi/execution_api/routes/__init__.py | 2 +
airflow/api_fastapi/execution_api/routes/health.py | 2 +-
.../execution_api/routes/task_instance.py | 131 +
airflow/api_fastapi/execution_api/schemas.py | 114 +
airflow/cli/cli_config.py | 18 -
airflow/cli/commands/dag_command.py | 1 -
airflow/cli/commands/dag_processor_command.py | 1 -
airflow/cli/commands/scheduler_command.py | 4 +-
airflow/cli/commands/task_command.py | 27 +-
airflow/dag_processing/manager.py | 28 +-
airflow/dag_processing/processor.py | 32 +-
.../example_branch_operator_decorator.py | 4 +-
airflow/example_dags/plugins/event_listener.py | 5 +-
airflow/executors/base_executor.py | 3 -
airflow/executors/debug_executor.py | 1 -
airflow/executors/local_executor.py | 1 -
airflow/executors/sequential_executor.py | 2 -
airflow/jobs/local_task_job_runner.py | 2 -
airflow/jobs/scheduler_job_runner.py | 32 +-
...metable_.py => 0046_3_0_0_drop_dag_pickling.py} | 46 +-
.../versions/0047_3_0_0_add_dag_versioning.py | 158 +
airflow/models/__init__.py | 4 +-
airflow/models/backfill.py | 6 +-
airflow/models/dag.py | 70 +-
airflow/models/dag_version.py | 167 +
airflow/models/dagbag.py | 3 +-
airflow/models/dagcode.py | 117 +-
airflow/models/dagpickle.py | 56 -
airflow/models/dagrun.py | 22 +-
airflow/models/serialized_dag.py | 141 +-
airflow/models/taskinstance.py | 70 +-
airflow/models/taskinstancehistory.py | 2 +
airflow/serialization/pydantic/dag.py | 2 -
airflow/serialization/pydantic/dag_run.py | 2 +-
airflow/serialization/pydantic/taskinstance.py | 2 -
airflow/serialization/schema.json | 1 +
airflow/serialization/serialized_objects.py | 4 -
airflow/task/standard_task_runner.py | 1 -
airflow/ui/openapi-gen/queries/common.ts | 127 +
airflow/ui/openapi-gen/queries/prefetch.ts | 165 +
airflow/ui/openapi-gen/queries/queries.ts | 233 ++
airflow/ui/openapi-gen/queries/suspense.ts | 193 +
airflow/ui/openapi-gen/requests/schemas.gen.ts | 315 +-
airflow/ui/openapi-gen/requests/services.gen.ts | 172 +
airflow/ui/openapi-gen/requests/types.gen.ts | 251 +-
airflow/ui/rules/react.js | 11 -
airflow/ui/src/components/DagRunInfo.tsx | 80 +
airflow/ui/src/components/Time.tsx | 13 +-
airflow/ui/src/pages/DagsList/Dag/Header.tsx | 24 +-
airflow/ui/src/pages/DagsList/DagCard.test.tsx | 2 -
airflow/ui/src/pages/DagsList/DagCard.tsx | 129 +-
airflow/ui/src/pages/DagsList/DagsList.tsx | 22 +-
airflow/utils/cli.py | 16 -
airflow/utils/db.py | 2 +-
airflow/utils/state.py | 9 +
airflow/www/package.json | 2 +-
airflow/www/static/css/main.css | 18 +
airflow/www/static/js/dags.js | 79 +-
airflow/www/static/js/main.js | 91 +-
airflow/www/static/js/types/api-generated.ts | 7 -
airflow/www/templates/airflow/main.html | 4 +-
airflow/www/views.py | 15 +-
airflow/www/webpack.config.js | 9 +-
airflow/www/yarn.lock | 19 +-
chart/dockerfiles/pgbouncer/Dockerfile | 4 +-
chart/dockerfiles/pgbouncer/build_and_push.sh | 6 +-
chart/values.schema.json | 2 +-
chart/values.yaml | 2 +-
dev/README_RELEASE_PROVIDER_PACKAGES.md | 1 -
dev/breeze/README.md | 72 +-
dev/breeze/doc/ci/02_images.md | 2 +-
dev/breeze/pyproject.toml | 15 +-
.../commands/release_candidate_command.py | 3 -
.../commands/release_management_commands.py | 10 +-
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
.../templates/pyproject_TEMPLATE.toml.jinja2 | 3 +-
.../src/airflow_breeze/utils/python_versions.py | 14 -
dev/breeze/src/airflow_breeze/utils/reinstall.py | 44 +-
.../src/airflow_breeze/utils/reproducible.py | 3 -
dev/breeze/src/airflow_breeze/utils/run_tests.py | 3 +-
.../tests/test_pytest_args_for_test_types.py | 2 +-
dev/breeze/uv.lock | 1904 ++++++++++
dev/perf/scheduler_dag_execution_timing.py | 4 +-
dev/perf/sql_queries.py | 2 +-
.../configurations-ref.rst | 0
docs/apache-airflow-providers-standard/index.rst | 1 +
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 3928 ++++++++++----------
docs/apache-airflow/migrations-ref.rst | 6 +-
generated/provider_metadata.json | 4 +
hatch_build.py | 3 +-
newsfragments/43612.significant.rst | 8 +
newsfragments/aip-72.significant.rst | 4 +
.../src/airflow/providers/amazon/aws/hooks/glue.py | 6 +-
.../airflow/providers/amazon/aws/operators/ecs.py | 6 +-
.../airflow/providers/amazon/aws/operators/glue.py | 11 +-
.../amazon/aws/transfers/s3_to_dynamodb.py | 2 +-
.../airflow/providers/amazon/aws/triggers/glue.py | 2 +-
.../celery/executors/celery_kubernetes_executor.py | 10 +-
.../providers/cncf/kubernetes/CHANGELOG.rst | 8 +
.../cncf/kubernetes/decorators/kubernetes.py | 2 +-
.../executors/local_kubernetes_executor.py | 9 +-
.../providers/cncf/kubernetes/provider.yaml | 1 +
.../providers/common/compat/assets/__init__.py | 5 +-
.../providers/fab/auth_manager/fab_auth_manager.py | 15 +-
.../providers/openlineage/extractors/manager.py | 2 +
.../airflow/providers/openlineage/utils/utils.py | 24 +-
.../src/airflow/providers/standard/provider.yaml | 17 +
.../providers/standard/utils/python_virtualenv.py | 84 +-
providers/tests/amazon/aws/operators/test_glue.py | 25 +
providers/tests/amazon/aws/triggers/test_glue.py | 18 +
.../tests/celery/executors/test_celery_executor.py | 3 -
.../cncf/kubernetes/decorators/test_kubernetes.py | 2 +
.../executors/test_kubernetes_executor.py | 3 -
.../api_endpoints/test_backfill_endpoint.py | 269 --
.../api_endpoints/test_dag_run_endpoint.py | 1 +
.../fab/auth_manager/test_fab_auth_manager.py | 26 +-
providers/tests/openlineage/plugins/test_utils.py | 125 +-
.../tests/standard/utils/test_python_virtualenv.py | 97 +-
.../ci/pre_commit/check_ti_vs_tis_attributes.py | 1 +
task_sdk/src/airflow/sdk/definitions/dag.py | 6 +
tests/api_connexion/endpoints/test_dag_endpoint.py | 32 -
.../endpoints/test_dag_run_endpoint.py | 5 +-
.../api_connexion/endpoints/test_task_endpoint.py | 6 +-
tests/api_connexion/schemas/test_dag_schema.py | 3 -
.../core_api/routes/public/test_connections.py | 119 +
.../core_api/routes/public/test_dag_stats.py} | 294 +-
.../core_api/routes/public/test_dags.py | 4 -
.../core_api/routes/public/test_event_logs.py | 130 +
.../core_api/routes/public/test_import_error.py | 219 ++
.../core_api/routes/public/test_task_instances.py | 6 +
.../api_fastapi/execution_api}/conftest.py | 7 +-
.../execution_api/routes/test_task_instance.py | 194 +
tests/cli/commands/test_task_command.py | 32 +-
tests/dag_processing/test_job_runner.py | 112 +-
tests/dag_processing/test_processor.py | 12 +-
tests/executors/test_base_executor.py | 4 -
tests/executors/test_local_executor.py | 3 -
tests/executors/test_sequential_executor.py | 3 -
tests/jobs/test_scheduler_job.py | 219 +-
tests/listeners/test_dag_import_error_listener.py | 2 +-
tests/models/test_dag.py | 31 +-
tests/models/test_dag_version.py | 113 +
tests/models/test_dagbag.py | 14 +-
tests/models/test_dagcode.py | 110 +-
tests/models/test_dagrun.py | 1 +
tests/models/test_serialized_dag.py | 111 +-
tests/models/test_taskinstance.py | 4 +-
tests/operators/test_trigger_dagrun.py | 13 +-
tests/sensors/test_external_task_sensor.py | 3 +-
tests/utils/test_cli_util.py | 18 +-
tests/utils/test_db_cleanup.py | 2 +-
tests/www/views/test_views_home.py | 2 +-
tests/www/views/test_views_tasks.py | 36 +-
tests_common/pytest_plugin.py | 59 +-
tests_common/test_utils/db.py | 1 +
200 files changed, 9718 insertions(+), 3756 deletions(-)
copy dev/breeze/src/airflow_breeze/commands/release_management_group.py =>
airflow/api_fastapi/common/db/dag_runs.py (74%)
copy tests/models/__init__.py => airflow/api_fastapi/common/types.py (78%)
create mode 100644 airflow/api_fastapi/core_api/routes/public/dag_stats.py
create mode 100644 airflow/api_fastapi/core_api/routes/public/import_error.py
copy airflow/api_fastapi/core_api/serializers/{dag_warning.py => dag_stats.py}
(67%)
copy airflow/api_fastapi/core_api/serializers/{event_logs.py =>
import_error.py} (70%)
rename airflow/api_fastapi/core_api/serializers/{ => ui}/dashboard.py (100%)
create mode 100644 airflow/api_fastapi/execution_api/routes/task_instance.py
create mode 100644 airflow/api_fastapi/execution_api/schemas.py
copy
airflow/migrations/versions/{0030_3_0_0_rename_schedule_interval_to_timetable_.py
=> 0046_3_0_0_drop_dag_pickling.py} (55%)
create mode 100644 airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py
create mode 100644 airflow/models/dag_version.py
delete mode 100644 airflow/models/dagpickle.py
create mode 100644 airflow/ui/src/components/DagRunInfo.tsx
create mode 100644 dev/breeze/uv.lock
copy docs/{apache-airflow-providers-amazon =>
apache-airflow-providers-standard}/configurations-ref.rst (100%)
create mode 100644 newsfragments/43612.significant.rst
delete mode 100644
providers/tests/fab/auth_manager/api_endpoints/test_backfill_endpoint.py
copy tests/{api_connexion/endpoints/test_dag_stats_endpoint.py =>
api_fastapi/core_api/routes/public/test_dag_stats.py} (61%)
create mode 100644
tests/api_fastapi/core_api/routes/public/test_import_error.py
copy {providers/tests/system/google =>
tests/api_fastapi/execution_api}/conftest.py (84%)
create mode 100644 tests/api_fastapi/execution_api/routes/test_task_instance.py
create mode 100644 tests/models/test_dag_version.py
