This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-7-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 84ec54af92 Update RELEASE_NOTES.rst for 2.7.0
add bc1d7b03ab Fix future DagRun rarely triggered by race conditions when
max_active_runs reached its upper limit. (#31414)
add c11e81460d Fix reload gunicorn workers (#32102)
add 0655d883ae Skip served logs for non-running task try (#32561)
add 097d2bed4e Replace State by TaskInstanceState in Airflow executors
(#32627)
add 699fba373d add dag_run_ids and task_ids filter for the batch task
instance API endpoint (#32705)
add b501446a2d Pass app context to webserver_config.py (#32759)
add 0ed56c84ef Add example for list-import-errors as a CI check (#32811)
add abfe7b89ef Documentation Update to enhance Readability (#32832)
add a887f92f69 Add links to DAGRun / DAG / Task in templates-ref.rst
(#33013)
add 41d57f0908 Add links to Celery and Kubernetes executor CLI commands to
Airflow (#33143)
add fdb96229b6 Configurable health check threshold for triggerer (#33089)
add 448f4849c0 Correctly log when there are problems trying to chmod
airflow.cfg (#33118)
add 1109ea9003 aDd documentation generation for CLI commands from
executors (#33081)
add 2b312cd162 Add notes about pre-2-7 config and cli options in providers
(#33123)
add dbacacbd4d Allows to choose SSL context for SMTP provider (#33075)
add 38fc9cd823 Allows to choose SSL context for IMAP provider (#33108)
add 11d6b3c864 openlineage, bigquery: add openlineage method support for
BigQueryExecuteQueryOperator (#31293)
add 2f894ca218 openlineage: disable running listener if not configured
(#33120)
add 7af9938a92 Move openlineage configuration to provider (#33124)
add 855d4684e9 Clean `schedule_interval` usages from example dags (#33131)
add b54f503037 Fix edge cases of "migrate/create-default-connections"
(#33136)
add 86f1dc100d Split PROD image verification to a separate step in CI
(#33140)
add c80d91be62 Attempt to stabilise tests for xcom_arg_map (#33150)
add f90625bb8b docs(public airflow interface): fix link (#33147)
add 0253f6aa1b Improve back-reference generation (#33149)
add 489d3dd6e9 Give the the test_xcom_ar_map test opportunity to flush the
data (#33153)
add 127445f577 Upgrade ruff to latest 0.0.282 version in pre-commits
(#33152)
add 197789e19d Give the the test_xcom_ar_map test opportunity to flush the
data (#33153)
add cf9e3fcddd Remove redundant dict.keys() call (#33158)
add 99aaacbd64 Refactor: Simplify dict manipulation in airflow/cli (#33159)
add 473452d3b7 Add elasticsearch group to pre-2.7 defaults (#33166)
add 816d4ccfde Fix installing older airflow versions in Breeze with
openssl fix (#33171)
add 39c3b5b62c Docs typo and capitalization fixes (#33176)
add 0eb0a7a70b Fixing typo in Dockerfile (#33180)
add 78f4344e0b Refactor: Simplify code in models (#33181)
add 690f3f1216 Cleanup all sqlalchemy sessions before each test (#33190)
add 7463380047 Fix xcom view returning bytes as xcom value (#33202)
add 29dbd32657 Don't ignore setups when arrowing from group (#33097)
add 9536fad87b For now limit Pydantic to < 2.0.0 (#33235)
add 28e88f412a Fix venv detection for Python virtualenv operator (#33223)
add af98c60554 Fix mistakenly used "release-mangement build-docs" command
(#33195)
add 39e590c04e Add graph screenshots in setup-teardown howto doc (#33244)
add 6378943620 Don't just default to scheduler heartbeat in jobs (#33084)
add a57cc8b0c0 Fix missing related PRs when cherry-picking (#33261)
add 494edad3e9 Enable pools to consider deferred tasks (#32709)
add e9c82d9319 wOrkaround failing dedlock when running backfill (#32991)
add 4e817c46b4 Fix flaky sqlite tests with `test_xcom_map_nest` hopefully
(#33145)
add 4e00fe8ec0 Refactor: Simplify code in dag_processing (#33161)
add 14ee535270 Gantt chart: Use earliest/oldest ti dates if different than
dag run start/end (#33215)
add b099803893 Remove add_task from TaskGroup (#33262)
add 2ff8d4d613 Change type of defualts for include_deferred (#33271)
add a050a6a72d Ensure include_deferred is not nullable (#33280)
add ba2a2db7bc Add a cache to `Variable` and `Connection` when called at
dag parsing time (#30259)
add 60e7f9d976 Update version in experimental Variable cache (#33286)
add 84a5e464dc Update RELEASE_NOTES.rst
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 14 +-
.pre-commit-config.yaml | 2 +-
Dockerfile | 2 +-
Dockerfile.ci | 1 -
RELEASE_NOTES.rst | 37 +++-
airflow/api/client/api_client.py | 3 +-
airflow/api/client/json_client.py | 9 +-
airflow/api/client/local_client.py | 10 +-
airflow/api/common/experimental/pool.py | 2 +-
airflow/api_connexion/endpoints/pool_endpoint.py | 8 +-
.../endpoints/task_instance_endpoint.py | 2 +
airflow/api_connexion/openapi/v1.yaml | 35 ++-
airflow/api_connexion/schemas/pool_schema.py | 8 +
.../api_connexion/schemas/task_instance_schema.py | 2 +
airflow/cli/cli_config.py | 18 +-
airflow/cli/cli_parser.py | 5 +-
airflow/cli/commands/db_command.py | 2 +-
airflow/cli/commands/pool_command.py | 18 +-
airflow/cli/commands/provider_command.py | 4 +-
airflow/cli/commands/role_command.py | 4 +-
airflow/cli/commands/webserver_command.py | 16 +-
airflow/cli/simple_table.py | 8 +-
airflow/config_templates/config.yml | 110 ++++------
airflow/config_templates/config.yml.schema.json | 29 ++-
airflow/config_templates/pre_2_7_defaults.cfg | 12 ++
airflow/config_templates/unit_tests.cfg | 10 +-
airflow/configuration.py | 11 +-
airflow/dag_processing/manager.py | 13 +-
airflow/dag_processing/processor.py | 12 +-
airflow/example_dags/example_python_operator.py | 5 +-
.../tutorial_taskflow_api_virtualenv.py | 4 +-
airflow/executors/base_executor.py | 25 ++-
airflow/executors/debug_executor.py | 28 +--
airflow/executors/sequential_executor.py | 6 +-
airflow/jobs/backfill_job_runner.py | 117 ++++++----
airflow/jobs/job.py | 13 +-
airflow/jobs/scheduler_job_runner.py | 21 +-
airflow/jobs/triggerer_job_runner.py | 4 +-
...8_2_7_0_add_include_deferred_column_to_pool.py} | 32 +--
airflow/models/abstractoperator.py | 8 -
airflow/models/base.py | 2 +-
airflow/models/baseoperator.py | 2 +-
airflow/models/connection.py | 11 +
airflow/models/dag.py | 35 ++-
airflow/models/dagbag.py | 2 +-
airflow/models/expandinput.py | 2 +-
airflow/models/pool.py | 58 ++++-
airflow/models/taskmixin.py | 7 +-
airflow/models/trigger.py | 14 +-
airflow/models/variable.py | 24 ++-
airflow/models/xcom_arg.py | 10 -
airflow/operators/python.py | 14 +-
.../providers/celery/executors/celery_executor.py | 17 +-
.../kubernetes/executors/kubernetes_executor.py | 18 +-
.../executors/kubernetes_executor_types.py | 5 +-
.../executors/kubernetes_executor_utils.py | 14 +-
airflow/providers/google/cloud/hooks/bigquery.py | 2 +-
.../providers/google/cloud/operators/bigquery.py | 92 +++++++-
airflow/providers/imap/CHANGELOG.rst | 12 ++
airflow/providers/imap/hooks/imap.py | 37 +++-
airflow/providers/imap/provider.yaml | 23 ++
airflow/providers/openlineage/extractors/base.py | 6 +
.../providers/openlineage/plugins/openlineage.py | 10 +-
airflow/providers/openlineage/provider.yaml | 61 ++++++
airflow/providers/openlineage/utils/utils.py | 9 +-
airflow/providers/qubole/hooks/qubole.py | 6 +-
airflow/providers/smtp/CHANGELOG.rst | 12 ++
airflow/providers/smtp/hooks/smtp.py | 20 +-
airflow/providers/smtp/provider.yaml | 24 +++
airflow/secrets/cache.py | 125 +++++++++++
airflow/ti_deps/deps/pool_slots_available_dep.py | 11 +-
airflow/utils/db.py | 2 +-
airflow/utils/edgemodifier.py | 9 -
airflow/utils/log/file_task_handler.py | 12 +-
airflow/utils/task_group.py | 34 +--
airflow/www/app.py | 4 +-
.../static/js/dag/details/gantt/GanttTooltip.tsx | 3 +
airflow/www/static/js/dag/details/gantt/Row.tsx | 20 +-
airflow/www/static/js/dag/details/gantt/index.tsx | 32 ++-
airflow/www/static/js/types/api-generated.ts | 26 ++-
airflow/www/views.py | 35 ++-
dev/airflow-github | 2 +-
.../airflow_breeze/commands/developer_commands.py | 47 +++-
.../commands/release_candidate_command.py | 5 +-
.../commands/release_management_commands.py | 32 ++-
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
.../airflow_breeze/utils/add_back_references.py | 71 +++---
.../src/airflow_breeze/utils/common_options.py | 4 +-
dev/breeze/tests/test_global_constants.py | 6 +-
dev/perf/sql_queries.py | 1 -
.../publish_provider_documentation.sh | 3 +-
.../connections/aws.rst | 2 +-
.../cli-ref.rst} | 20 +-
docs/apache-airflow-providers-celery/index.rst | 1 +
.../cli-ref.rst} | 15 +-
.../index.rst | 1 +
.../configurations-ref.rst | 0
docs/apache-airflow-providers-imap/index.rst | 1 +
.../configurations-ref.rst | 0
.../apache-airflow-providers-openlineage/index.rst | 1 +
.../configurations-ref.rst | 0
docs/apache-airflow-providers-smtp/index.rst | 1 +
.../dag-serialization.rst | 2 +-
.../logging-monitoring/metrics.rst | 1 +
.../administration-and-deployment/pools.rst | 2 +-
.../authoring-and-scheduling/deferring.rst | 2 +-
.../authoring-and-scheduling/plugins.rst | 6 +-
docs/apache-airflow/best-practices.rst | 23 +-
docs/apache-airflow/cli-and-env-variables-ref.rst | 6 +
docs/apache-airflow/configurations-ref.rst | 3 +
docs/apache-airflow/howto/add-dag-tags.rst | 4 +-
docs/apache-airflow/howto/set-config.rst | 12 ++
docs/apache-airflow/howto/setup-and-teardown.rst | 71 ++++--
docs/apache-airflow/howto/usage-cli.rst | 31 +++
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 78 +++----
docs/apache-airflow/img/setup-teardown-complex.png | Bin 0 -> 114476 bytes
docs/apache-airflow/img/setup-teardown-group.png | Bin 0 -> 48789 bytes
docs/apache-airflow/img/setup-teardown-nesting.png | Bin 0 -> 83934 bytes
.../apache-airflow/img/setup-teardown-parallel.png | Bin 0 -> 43571 bytes
docs/apache-airflow/img/setup-teardown-scope.png | Bin 0 -> 62951 bytes
.../img/setup-teardown-setup-group.png | Bin 0 -> 76759 bytes
docs/apache-airflow/img/setup-teardown-simple.png | Bin 0 -> 29606 bytes
docs/apache-airflow/migrations-ref.rst | 4 +-
docs/apache-airflow/public-airflow-interface.rst | 2 +-
docs/apache-airflow/security/webserver.rst | 27 +++
docs/apache-airflow/templates-ref.rst | 16 +-
docs/apache-airflow/ui.rst | 2 +-
docs/docker-stack/build.rst | 2 +-
.../exts/includes/providers-configurations-ref.rst | 6 +
docs/exts/includes/sections-and-options.rst | 31 +--
docs/helm-chart/airflow-configuration.rst | 9 -
images/breeze/output-commands-hash.txt | 4 +-
...tput_release-management_add-back-references.svg | 108 ++++++++--
scripts/ci/pre_commit/pre_commit_insert_extras.py | 4 +-
.../ci/pre_commit/pre_commit_local_yml_mounts.py | 6 +-
scripts/docker/entrypoint_ci.sh | 1 -
scripts/docker/entrypoint_prod.sh | 2 +-
scripts/in_container/_in_container_utils.sh | 13 +-
scripts/in_container/check_environment.sh | 18 +-
setup.cfg | 5 +-
tests/always/test_secrets.py | 8 +
.../api_connexion/endpoints/test_pool_endpoint.py | 120 ++++++++---
.../endpoints/test_task_instance_endpoint.py | 30 +++
tests/api_connexion/schemas/test_pool_schemas.py | 14 +-
tests/api_connexion/test_auth.py | 4 +
tests/api_experimental/client/test_local_client.py | 20 +-
.../common/experimental/test_pool.py | 7 +-
tests/cli/commands/test_pool_command.py | 67 ++++--
tests/cli/commands/test_task_command.py | 2 +-
tests/conftest.py | 9 +
tests/core/test_configuration.py | 1 -
tests/jobs/test_backfill_job.py | 5 +-
tests/jobs/test_base_job.py | 42 +++-
tests/jobs/test_scheduler_job.py | 31 +--
tests/models/test_pool.py | 70 +++++-
tests/models/test_taskinstance.py | 2 +-
tests/models/test_trigger.py | 86 +++-----
tests/models/test_variable.py | 39 ++++
tests/models/test_xcom_arg_map.py | 36 ++--
tests/operators/test_python.py | 6 +-
tests/operators/test_subdag_operator.py | 8 +-
tests/providers/daskexecutor/test_dask_executor.py | 1 -
.../google/cloud/operators/job_details.json | 240 +++++++++++++++++++++
.../google/cloud/operators/test_bigquery.py | 91 ++++++++
tests/providers/imap/hooks/test_imap.py | 69 +++++-
.../openlineage/extractors/test_bash_extractor.py | 2 +-
.../extractors/test_python_extractor.py | 2 +-
.../openlineage/plugins/test_openlineage.py | 36 +++-
tests/providers/smtp/hooks/test_smtp.py | 103 ++++++++-
tests/secrets/test_cache.py | 142 ++++++++++++
.../apache/kafka/example_dag_hello_kafka.py | 2 +-
tests/system/providers/ftp/example_ftp.py | 2 +-
.../google/cloud/compute/example_compute.py | 2 +-
.../google/cloud/compute/example_compute_igm.py | 2 +-
.../google/cloud/compute/example_compute_ssh.py | 2 +-
.../dataproc/example_dataproc_spark_deferrable.py | 2 +-
.../ti_deps/deps/test_pool_slots_available_dep.py | 13 +-
tests/ti_deps/deps/test_runnable_exec_date_dep.py | 8 +-
tests/utils/test_helpers.py | 8 +
tests/utils/test_log_handlers.py | 16 +-
tests/utils/test_task_group.py | 120 ++++++++++-
tests/www/api/experimental/test_endpoints.py | 2 +
tests/www/views/test_views_pool.py | 5 +
tests/www/views/test_views_tasks.py | 23 +-
185 files changed, 2938 insertions(+), 812 deletions(-)
copy
airflow/migrations/versions/{0127_2_7_0_add_custom_operator_name_column.py =>
0128_2_7_0_add_include_deferred_column_to_pool.py} (53%)
create mode 100644 airflow/secrets/cache.py
copy docs/{apache-airflow-providers-apache-kafka/triggers.rst =>
apache-airflow-providers-celery/cli-ref.rst} (66%)
copy docs/{apache-airflow-providers-apache-hdfs/operators/index.rst =>
apache-airflow-providers-cncf-kubernetes/cli-ref.rst} (66%)
copy docs/{apache-airflow-providers-apache-hive =>
apache-airflow-providers-imap}/configurations-ref.rst (100%)
copy docs/{apache-airflow-providers-apache-hive =>
apache-airflow-providers-openlineage}/configurations-ref.rst (100%)
copy docs/{apache-airflow-providers-apache-hive =>
apache-airflow-providers-smtp}/configurations-ref.rst (100%)
create mode 100644 docs/apache-airflow/img/setup-teardown-complex.png
create mode 100644 docs/apache-airflow/img/setup-teardown-group.png
create mode 100644 docs/apache-airflow/img/setup-teardown-nesting.png
create mode 100644 docs/apache-airflow/img/setup-teardown-parallel.png
create mode 100644 docs/apache-airflow/img/setup-teardown-scope.png
create mode 100644 docs/apache-airflow/img/setup-teardown-setup-group.png
create mode 100644 docs/apache-airflow/img/setup-teardown-simple.png
create mode 100644 tests/providers/google/cloud/operators/job_details.json
create mode 100644 tests/secrets/test_cache.py
