This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-7-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
from c08c82e9dd fixup! Update RELEASE_NOTES.rst
add 126b507e82 Remove openlineage exclusion (#33491)
add dc44806dc3 Fix OpenLineage link in New Features notes (#33513)
add a335ca5a58 Update the release date for 2.7.0
add 100f37d9a4 Remove Pydantic limitation for version < 2 (#33507)
add 698ad804d1 Add MySQL 8.1 to supported versions. (#33576)
add 3fad890393 Fix dependencies for celery and opentelemetry for Python
3.8 (#33579)
add 6005da9ba5 Improve detection of when breeze CI image needs rebuilding
(#33603)
add fdd1bc374c Upgrade botocore/aiobotocore minimum requirements (#33649)
add 45d228239a Limit hive provider check for Python 3.11 temporarily
(#33774)
add 20bfcafa31 Limit Redis dependencies to match celery limits (#33773)
add d39abde825 Remove "eager upgrade" from PROD image completely (#33784)
add 529346b6ad Bump airflow version to 2.7.1
add b3bb92a461 Use PyPI constraints for PROD image in non-main branch
(#33789)
add 22d09d30df Fix rendering the mapped parameters when using
`expand_kwargs` method (#32272)
add 56ec8d7805 Bring back hive support for Python 3.11 (#32607)
add fc77886fd5 [Models] [Postgres] Check if the dynamically-added index is
in the table schema before adding (#32731)
add 34aef4be89 Fix getting correct commit from multiple referenced PR
(#33411)
add 04e94bf2b4 More complete fix for dev release scripts to filter commit
for PR (#33418)
add aab0f4b4af Fix handling of default value and serialization of Param
class (#33141)
add ee20c88aa3 Refactor: Simplify code in configuration.py (#33160)
add 11845bb042 respect soft_fail argument when ExternalTaskSensor runs in
deferrable mode (#33196)
add bc9ea43753 Fix pydantic warning about `orm_mode` rename (#33220)
add 84369c6291 Refactor: Simplify dict manipulation in metrics (#33264)
add d31f20b3c0 Refactor: Simplify code in serialization (#33266)
add 3439f91325 Refactor: Simplify code in settings (#33267)
add cff420b3c3 Refactor: Simplify code in utils (#33268)
add 496f917f1a Refactor: Simplify code in www (#33270)
add 4e4911b590 Refactor: Simplify code in scripts (#33295)
add d329d70632 Refactor str.startswith with tuples (#33292)
add c597ca1e8b Fix test_example_dags (#32714)
add 75bf02ebd3 Refactor: Simplify code in tests (#33293)
add 40743706d1 D205 Support - Root files (#33297)
add a33cbf0355 D205 Support - WWW (#33298)
add 601d0e41cc D401 Support - Secrets to Triggers (Inclusive) (#33338)
add 4af3afd567 D205 Support - Stragglers (#33301)
add bb6ea0f298 Remove the replace to <br> tag in Provider's view (#33326)
add 548e3f55b3 D401 Support - Airflow/api thru Airflow/auth (#33333)
add f7e39af87d Fix some missing type hints (#33334)
add 448cf69700 D401 Support - airflow/callbacks thru airflow/decorators
(#33335)
add 85d75a2f78 D401 Support - airflow/example_dags thru airflow/listeners
(#33336)
add 4c0158440f D401 Support - Macros to Operators (Inclusive) (#33337)
add 5988ea6b0c D401 Support - Utils (#33339)
add 14c91db04a D401 Support - Root Files (#33352)
add f44a3246d1 Fix DagFileProcessor interfering with dags outside its
``processor_subdir`` (#33357)
add c3580fc4dd Fix config description for base_log_folder (#33388)
add 08d4e0168c Simplify 'X for X in Y' to 'Y' where applicable (#33453)
add 6996d308a6 Create a new method used to resume the task in order to
implement specific logic for operators (#33424)
add dfe129f77c Respect "soft_fail" argument when "poke" is called (#33401)
add a597d5eb8b Allow timetable to slightly miss catchup cutoff (#33404)
add 0fbb01b5b9 Import utc from datetime and normalize its import (#33450)
add 24b11c9950 make `conf.set` case insensitive (#33452)
add 5930e18908 Fix Cluster Activity Health margin (#33456)
add f5ea787b3e Revert "respect soft_fail argument when ExternalTaskSensor
runs in deferrable mode (#33196)" (#33458)
add 00fe5465a1 Static check fixes (#33462)
add ab1fd50ef0 Fix secrets backend docs (#33471)
add 29ffb32057 Use absolute path in README.md for broken links on pypi
listing (#33473)
add 502909253e Simplify code around enumerate (#33476)
add 20ebd990a7 Consolidate import and usage of pandas (#33480)
add ea432df49a Fix a bug in formatDuration method (#33486)
add 5c8e799d92 Do not return ongoin dagrun when a end_date is less than
utcnow (#33488)
add e2578691ba Fix broken link in Modules Management page (#33499)
add 7eeba7c3a6 Use `dialect.name` in custom SA types (#33503)
add 3f8c35246f Update version_added field for configs in config file
(#33509)
add dc73e786fe Fix typo in `db upgrade` warning message (#33510)
add a3fa682e8c Set strict to True when parsing dates in webserver views
(#33512)
add f4c2b58ec8 Move license templates out of repo root (#33515)
add df352f37e7 Stop adding values to rendered templates UI when there is
no dagrun (#33516)
add 42cfb9143f Fix typo in release notes (#33521)
add 5101840c3e Sort data before groupby in TIS duration calculation
(#33535)
add d508a9a54b Add a fallback in case no first name and last name are set
(#33617)
add e77f83dbd9 Bind engine before attempting to drop archive tables
(#33622)
add e6f6dea2b1 Treat dag-defined access_control as authoritative if
defined (#33632)
add b0f9afb2b5 Set context inside templates (#33645)
add 4befd33b37 Fix 2.7.0 db migration job errors (#33652)
add 37ac1e8728 Add back `get_url_for_login` in security manager (#33660)
add 2ebfe475a1 SECURITY_MANAGER_CLASS should be a referrence to class, not
a string (#33690)
add 12db5ed16e Fix cleaning zombie RESTARTING tasks (#33706)
add e32036e8d2 Add documentation explaining template_ext (and how to
override it) (#33735)
add d83c0c511b Use f-string instead of in Airflow core (#33753)
add 69fc667ede Fix UI DAG counts including deleted DAGs (#33778)
add e793cf2eb5 Refactor: Simplofy code in dev (#33294)
add b214417c63 Refactor: Simplify code in providers/amazon (#33222)
add 5b213b81fb Refactor: Simplify code in smaller providers (#33234)
add 2c14bb45f7 Simplify conditions on len() in other providers (#33569)
add 2aa33449ed Refactor: Simplify code in providers/cncf (#33230)
add fe85781ceb Consolidate import and usage of itertools (#33479)
add dffc1d6488 Validate SqoopHook connection string and disable extra
options from public hook methods (#33039)
add 352c96e039 Replace OrderedDict with plain dict (#33508)
add 5727eb6e39 openlineage: do not try to redact Proxy objects from
deprecated config (#33393)
add 6587b75bda Replace repr() with proper formatting (#33520)
add f4557e53ea Simplify conditions on `len()` in utils (#33567)
add 64b9ebb089 Simplify conditions on len() in jobs (#33568)
add eb96b86e87 Automatically update the REVISIONS_HEADS_MAP in db.py
(#33616)
add 9314b4d77f Fix version heads map pre_commit rule (#33749)
add 8043bab9b2 Add limit 1 if required first value from query result
(#33672)
add 70bfb11904 Update version to 2.7.1
add 22df7b1112 Fix MappedTaskGroup tasks not respecting upstream
dependency (#33732)
add d254b5b367 Replace AIP-42 label with area:dynamic-task-mapping label
(#33870)
add 2d16c702d0 Raise 404 from Variable PATCH API if variable is not found
(#33885)
add 99a4f9f5df Add minimum requirement for sqlalchemy to 1.4.24 (#33892)
add a14177141f Clarify that DAG authors can also run code in DAG File
Processor (#33920)
add 71e842e274 Bumping FAB to 4.3.4 in order to fix issues with filters
(#33931)
add ebbebf14bb Differentiate 0 and unset as a default param values (#33965)
add 01e2b1696f Reinstall CI depdendencies from scratch (#33978)
add 8754b509bd Respect "soft_fail" for core async sensors (#33403)
add d0886260d7 fix(sensors): move trigger initialization from __init___ to
execute (#33926)
add f9d653bbc2 Suspend qubole provider (#33889)
add 4dd75306e3 Reorganize devel_only extra in airflow's setup.py (#33907)
add ae03ad5312 Upgrade to Pydantic 2 (#33956)
add b83fa2eba9 Explain the users how they can check if python code is
top-level (#34006)
add c874f0ff8b Use a trimmed version of README.md for PyPI (#33637)
add de95f98272 Upgrade Elasticsearch to 8 (#33135)
add 7cd0748c60 Remove print introduced in PR #32261 (#34008)
add d404d3f84a Limit celery by excluding 5.3.2 and 5.3.3 (#34031)
add 590412ba58 Fix typos (double words and it's/its) (#33623)
add b5dbf97317 Ensure that tasks wait for running indirect setup (#33903)
add 891fae5025 Update RELEASE_NOTES.rst
No new revisions were added by this update.
Summary of changes:
.github/actions/build-ci-images/action.yml | 14 +-
.github/actions/build-prod-images/action.yml | 21 +-
.github/workflows/build-images.yml | 2 +-
.pre-commit-config.yaml | 31 +-
CONTRIBUTING.rst | 6 +-
CONTRIBUTORS_QUICK_START.rst | 4 +-
Dockerfile | 56 +-
Dockerfile.ci | 6 +-
INSTALL | 6 +-
README.md | 68 ++-
RELEASE_NOTES.rst | 115 ++++-
STATIC_CODE_CHECKS.rst | 2 +
airflow/__init__.py | 2 +-
airflow/__main__.py | 1 -
airflow/api_connexion/endpoints/config_endpoint.py | 1 -
.../endpoints/role_and_permission_endpoint.py | 2 +-
.../api_connexion/endpoints/variable_endpoint.py | 4 +-
airflow/api_connexion/exceptions.py | 2 +-
airflow/api_connexion/openapi/v1.yaml | 2 +-
airflow/api_connexion/parameters.py | 4 +-
airflow/api_connexion/schemas/dag_schema.py | 6 +-
airflow/api_connexion/schemas/pool_schema.py | 12 +-
.../api_connexion/schemas/task_instance_schema.py | 4 +-
airflow/api_connexion/security.py | 4 +-
airflow/api_internal/endpoints/rpc_api_endpoint.py | 2 +-
airflow/api_internal/internal_api_call.py | 6 +-
airflow/auth/managers/fab/models/__init__.py | 8 +-
airflow/callbacks/base_callback_sink.py | 2 +-
airflow/callbacks/database_callback_sink.py | 2 +-
airflow/callbacks/pipe_callback_sink.py | 2 +-
airflow/cli/cli_config.py | 6 +-
airflow/cli/cli_parser.py | 2 +-
airflow/cli/commands/celery_command.py | 8 +-
airflow/cli/commands/connection_command.py | 12 +-
airflow/cli/commands/dag_command.py | 26 +-
airflow/cli/commands/dag_processor_command.py | 4 +-
airflow/cli/commands/db_command.py | 57 +-
airflow/cli/commands/info_command.py | 6 +-
airflow/cli/commands/internal_api_command.py | 2 +-
airflow/cli/commands/jobs_command.py | 2 +-
airflow/cli/commands/kubernetes_command.py | 8 +-
airflow/cli/commands/legacy_commands.py | 2 +-
airflow/cli/commands/pool_command.py | 16 +-
airflow/cli/commands/provider_command.py | 20 +-
airflow/cli/commands/role_command.py | 12 +-
airflow/cli/commands/scheduler_command.py | 6 +-
airflow/cli/commands/standalone_command.py | 17 +-
airflow/cli/commands/sync_perm_command.py | 2 +-
airflow/cli/commands/task_command.py | 18 +-
airflow/cli/commands/triggerer_command.py | 4 +-
airflow/cli/commands/user_command.py | 12 +-
airflow/cli/commands/variable_command.py | 14 +-
airflow/cli/commands/version_command.py | 2 +-
airflow/cli/commands/webserver_command.py | 10 +-
airflow/cli/simple_table.py | 10 +-
airflow/config_templates/config.yml | 116 +----
airflow/configuration.py | 121 +++--
airflow/dag_processing/manager.py | 38 +-
airflow/dag_processing/processor.py | 4 +-
airflow/decorators/base.py | 6 +-
airflow/decorators/branch_python.py | 2 +-
airflow/decorators/external_python.py | 3 +-
airflow/decorators/python.py | 3 +-
airflow/decorators/python_virtualenv.py | 3 +-
airflow/decorators/sensor.py | 2 +-
airflow/decorators/short_circuit.py | 3 +-
airflow/example_dags/example_sensors.py | 19 +-
.../example_setup_teardown_taskflow.py | 2 +-
airflow/exceptions.py | 5 +-
airflow/executors/base_executor.py | 22 +-
airflow/executors/executor_loader.py | 13 +-
airflow/executors/local_executor.py | 22 +-
airflow/hooks/base.py | 6 +-
airflow/hooks/filesystem.py | 2 +-
airflow/hooks/subprocess.py | 2 +-
airflow/jobs/backfill_job_runner.py | 14 +-
airflow/jobs/base_job_runner.py | 12 +-
airflow/jobs/job.py | 12 +-
airflow/jobs/local_task_job_runner.py | 6 +-
airflow/jobs/scheduler_job_runner.py | 43 +-
airflow/jobs/triggerer_job_runner.py | 31 +-
.../pre_7_4_0_compatibility/kube_client.py | 4 +-
.../pre_7_4_0_compatibility/pod_generator.py | 13 +-
.../pod_generator_deprecated.py | 6 +-
.../kubernetes/pre_7_4_0_compatibility/secret.py | 6 +-
airflow/lineage/__init__.py | 7 +-
airflow/lineage/backend.py | 2 +-
airflow/listeners/spec/dagrun.py | 6 +-
airflow/listeners/spec/lifecycle.py | 4 +-
airflow/listeners/spec/taskinstance.py | 6 +-
airflow/metrics/otel_logger.py | 27 +-
airflow/metrics/statsd_logger.py | 2 +-
airflow/metrics/validators.py | 2 +-
airflow/migrations/utils.py | 2 +-
airflow/models/abstractoperator.py | 17 +-
airflow/models/baseoperator.py | 54 +-
airflow/models/connection.py | 16 +-
airflow/models/dag.py | 85 +--
airflow/models/dagbag.py | 8 +-
airflow/models/dagcode.py | 26 +-
airflow/models/dagrun.py | 42 +-
airflow/models/expandinput.py | 5 +-
airflow/models/mappedoperator.py | 6 +-
airflow/models/param.py | 15 +-
airflow/models/serialized_dag.py | 13 +-
airflow/models/skipmixin.py | 6 +-
airflow/models/taskinstance.py | 70 ++-
airflow/models/taskinstancekey.py | 2 +-
airflow/models/taskmixin.py | 12 +-
airflow/models/taskreschedule.py | 4 +-
airflow/models/trigger.py | 8 +-
airflow/models/variable.py | 8 +-
airflow/models/xcom.py | 10 +-
airflow/models/xcom_arg.py | 12 +-
airflow/notifications/basenotifier.py | 2 +-
airflow/operators/bash.py | 2 +-
airflow/operators/datetime.py | 2 +-
airflow/operators/python.py | 9 +-
airflow/plugins_manager.py | 1 +
airflow/policies.py | 10 +-
.../SUSPENDING_AND_RESUMING_PROVIDERS.rst | 2 +-
airflow/providers/amazon/aws/hooks/base_aws.py | 7 +-
airflow/providers/amazon/aws/hooks/batch_client.py | 91 ++--
airflow/providers/amazon/aws/hooks/datasync.py | 28 +-
airflow/providers/amazon/aws/hooks/quicksight.py | 2 +-
.../providers/amazon/aws/hooks/redshift_data.py | 6 +-
airflow/providers/amazon/aws/hooks/s3.py | 7 +-
airflow/providers/amazon/aws/hooks/sagemaker.py | 31 +-
.../providers/amazon/aws/log/s3_task_handler.py | 7 +-
airflow/providers/amazon/aws/operators/eks.py | 2 +-
.../amazon/aws/operators/redshift_cluster.py | 18 +-
.../amazon/aws/secrets/secrets_manager.py | 5 +-
.../amazon/aws/transfers/redshift_to_s3.py | 2 +-
.../amazon/aws/transfers/s3_to_redshift.py | 2 +-
.../providers/amazon/aws/transfers/sql_to_s3.py | 14 +-
airflow/providers/amazon/aws/triggers/README.md | 2 +-
airflow/providers/amazon/aws/triggers/batch.py | 8 +-
airflow/providers/amazon/aws/triggers/ecs.py | 4 +-
airflow/providers/amazon/aws/triggers/emr.py | 4 +-
airflow/providers/amazon/aws/triggers/sagemaker.py | 4 +-
.../amazon/aws/utils/waiter_with_logging.py | 30 +-
airflow/providers/amazon/provider.yaml | 22 +-
airflow/providers/apache/hdfs/CHANGELOG.rst | 2 +-
airflow/providers/apache/hive/CHANGELOG.rst | 1 +
airflow/providers/apache/hive/hooks/hive.py | 23 +-
.../providers/apache/hive/operators/hive_stats.py | 2 -
airflow/providers/apache/hive/provider.yaml | 11 +-
.../apache/hive/transfers/mssql_to_hive.py | 3 +-
.../apache/hive/transfers/mysql_to_hive.py | 3 +-
.../apache/hive/transfers/vertica_to_hive.py | 3 +-
.../apache/kafka/triggers/await_message.py | 2 +-
airflow/providers/apache/sqoop/CHANGELOG.rst | 11 +
airflow/providers/apache/sqoop/hooks/sqoop.py | 37 +-
airflow/providers/apache/sqoop/operators/sqoop.py | 30 +-
airflow/providers/apache/sqoop/provider.yaml | 1 +
airflow/providers/celery/provider.yaml | 3 +-
.../kubernetes/executors/kubernetes_executor.py | 4 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 4 +-
airflow/providers/cncf/kubernetes/operators/pod.py | 2 +-
airflow/providers/cncf/kubernetes/pod_generator.py | 2 +-
airflow/providers/cncf/kubernetes/triggers/pod.py | 11 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 20 +-
airflow/providers/common/sql/hooks/sql.pyi | 2 +-
airflow/providers/common/sql/operators/sql.py | 2 +-
airflow/providers/common/sql/operators/sql.pyi | 2 +-
airflow/providers/common/sql/sensors/sql.pyi | 2 +-
.../daskexecutor/executors/dask_executor.py | 2 +-
.../providers/databricks/hooks/databricks_sql.py | 2 +-
.../databricks/operators/databricks_sql.py | 2 +-
.../databricks/sensors/databricks_partition.py | 4 +-
airflow/providers/databricks/utils/databricks.py | 10 +-
airflow/providers/datadog/sensors/datadog.py | 2 +-
airflow/providers/dbt/cloud/hooks/dbt.py | 2 +-
airflow/providers/docker/operators/docker.py | 4 +-
airflow/providers/docker/operators/docker_swarm.py | 2 +-
airflow/providers/elasticsearch/CHANGELOG.rst | 6 +
.../providers/elasticsearch/log/es_task_handler.py | 19 +-
airflow/providers/elasticsearch/provider.yaml | 96 +++-
airflow/providers/google/cloud/hooks/bigquery.py | 4 +-
.../google/cloud/operators/kubernetes_engine.py | 2 +-
airflow/providers/google/cloud/operators/pubsub.py | 2 +-
.../providers/google/cloud/operators/workflows.py | 11 +-
airflow/providers/google/cloud/sensors/pubsub.py | 2 +-
.../google/cloud/triggers/bigquery_dts.py | 2 +-
.../google/cloud/triggers/kubernetes_engine.py | 2 +-
airflow/providers/google/cloud/triggers/pubsub.py | 2 +-
airflow/providers/grpc/operators/grpc.py | 2 +-
airflow/providers/hashicorp/hooks/vault.py | 2 +-
airflow/providers/http/hooks/http.py | 24 +-
airflow/providers/imap/hooks/imap.py | 2 +-
airflow/providers/microsoft/azure/CHANGELOG.rst | 4 +-
.../providers/microsoft/azure/hooks/data_lake.py | 2 +-
.../providers/microsoft/azure/operators/batch.py | 2 +-
.../microsoft/azure/operators/data_factory.py | 2 +-
airflow/providers/openlineage/utils/sql.py | 2 +-
airflow/providers/openlineage/utils/utils.py | 69 +--
.../providers/oracle/transfers/oracle_to_oracle.py | 6 +-
airflow/providers/papermill/operators/papermill.py | 6 +-
airflow/providers/plexus/operators/job.py | 2 +-
airflow/providers/presto/hooks/presto.py | 6 +-
airflow/providers/qubole/provider.yaml | 6 +-
airflow/providers/redis/provider.yaml | 6 +-
airflow/providers/sendgrid/utils/emailer.py | 4 +-
airflow/providers/slack/transfers/sql_to_slack.py | 4 +-
airflow/providers/smtp/hooks/smtp.py | 2 +-
airflow/providers/ssh/hooks/ssh.py | 2 +-
airflow/providers/trino/hooks/trino.py | 6 +-
airflow/providers_manager.py | 14 +-
airflow/secrets/cache.py | 18 +-
airflow/secrets/local_filesystem.py | 6 +-
airflow/security/permissions.py | 2 +-
airflow/security/utils.py | 6 +-
airflow/sensors/base.py | 20 +-
airflow/sensors/date_time.py | 10 +-
airflow/sensors/external_task.py | 9 +-
airflow/sensors/time_delta.py | 12 +-
airflow/sensors/time_sensor.py | 5 +-
airflow/sentry.py | 6 +-
airflow/serialization/enums.py | 1 +
airflow/serialization/helpers.py | 3 +-
airflow/serialization/pydantic/dag_run.py | 3 +-
airflow/serialization/pydantic/dataset.py | 12 +-
airflow/serialization/pydantic/job.py | 3 +-
airflow/serialization/pydantic/taskinstance.py | 3 +-
airflow/serialization/serde.py | 13 +-
airflow/serialization/serialized_objects.py | 44 +-
airflow/serialization/serializers/pandas.py | 8 +-
airflow/settings.py | 55 +-
airflow/task/task_runner/base_task_runner.py | 2 +-
airflow/template/templater.py | 11 +-
airflow/ti_deps/dep_context.py | 2 +-
airflow/ti_deps/deps/base_ti_dep.py | 6 +-
airflow/ti_deps/deps/dagrun_backfill_dep.py | 2 +-
airflow/ti_deps/deps/mapped_task_expanded.py | 2 +-
airflow/ti_deps/deps/not_previously_skipped_dep.py | 2 +-
airflow/ti_deps/deps/pool_slots_available_dep.py | 2 +-
airflow/ti_deps/deps/ready_to_reschedule.py | 2 +-
airflow/ti_deps/deps/trigger_rule_dep.py | 572 +++++++++++++--------
airflow/timetables/interval.py | 3 +-
airflow/timetables/trigger.py | 2 +-
airflow/triggers/base.py | 4 +-
airflow/triggers/external_task.py | 18 +-
airflow/triggers/file.py | 2 +-
airflow/utils/cli.py | 26 +-
airflow/utils/cli_action_loggers.py | 11 +-
airflow/utils/cli_app_builder.py | 2 +-
airflow/utils/code_utils.py | 4 +-
airflow/utils/configuration.py | 2 +-
airflow/utils/dag_cycle_tester.py | 2 +-
airflow/utils/dates.py | 4 +-
airflow/utils/db.py | 29 +-
airflow/utils/db_cleanup.py | 1 +
airflow/utils/decorators.py | 4 +-
airflow/utils/dot_renderer.py | 6 +-
airflow/utils/edgemodifier.py | 6 +-
airflow/utils/email.py | 6 +-
airflow/utils/empty_set.py | 2 +-
airflow/utils/event_scheduler.py | 2 +-
airflow/utils/file.py | 11 +-
airflow/utils/hashlib_wrapper.py | 3 +-
airflow/utils/helpers.py | 28 +-
airflow/utils/json.py | 2 +-
airflow/utils/log/file_task_handler.py | 15 +-
airflow/utils/log/log_reader.py | 6 +-
airflow/utils/log/logging_mixin.py | 12 +-
airflow/utils/log/secrets_masker.py | 18 +-
airflow/utils/operator_helpers.py | 2 +-
airflow/utils/platform.py | 2 +-
airflow/utils/process_utils.py | 6 +-
airflow/utils/providers_configuration_loader.py | 3 +-
airflow/utils/python_virtualenv.py | 5 +-
airflow/utils/retries.py | 3 +-
airflow/utils/serve_logs.py | 2 +-
airflow/utils/session.py | 2 +-
airflow/utils/setup_teardown.py | 17 +-
airflow/utils/sqlalchemy.py | 47 +-
airflow/utils/state.py | 10 +-
airflow/utils/task_group.py | 29 +-
airflow/utils/timeout.py | 4 +-
airflow/utils/timezone.py | 11 +-
airflow/utils/trigger_rule.py | 4 +-
airflow/utils/types.py | 2 +-
airflow/utils/weekday.py | 2 +-
airflow/utils/weight_rule.py | 2 +-
airflow/www/api/experimental/endpoints.py | 13 +-
airflow/www/decorators.py | 9 +-
airflow/www/extensions/init_appbuilder.py | 16 +-
airflow/www/extensions/init_dagbag.py | 5 +-
airflow/www/extensions/init_robots.py | 6 +-
airflow/www/extensions/init_security.py | 5 +-
airflow/www/fab_security/manager.py | 26 +-
airflow/www/fab_security/sqla/manager.py | 11 +-
airflow/www/forms.py | 5 +-
airflow/www/security.py | 48 +-
.../js/cluster-activity/live-metrics/Health.tsx | 1 +
.../static/js/cluster-activity/nav/FilterBar.tsx | 2 +-
.../www/static/js/cluster-activity/useFilters.tsx | 5 +-
airflow/www/static/js/datetime_utils.js | 3 +-
airflow/www/templates/airflow/trigger.html | 2 +-
airflow/www/templates/appbuilder/navbar_right.html | 9 +-
airflow/www/utils.py | 27 +-
airflow/www/views.py | 119 +++--
...UALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md | 2 +-
dev/README_RELEASE_AIRFLOW.md | 2 -
dev/README_RELEASE_PROVIDER_PACKAGES.md | 2 +-
dev/airflow-github | 23 +-
dev/airflow-license | 2 +-
dev/breeze/SELECTIVE_CHECKS.md | 6 +-
.../adr/0003-bootstrapping-virtual-environment.md | 6 +-
.../airflow_breeze/commands/ci_image_commands.py | 8 +-
.../commands/ci_image_commands_config.py | 1 +
.../airflow_breeze/commands/developer_commands.py | 3 +
.../commands/production_image_commands.py | 73 ++-
.../commands/production_image_commands_config.py | 4 +-
.../src/airflow_breeze/commands/setup_commands.py | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
.../src/airflow_breeze/params/build_ci_params.py | 6 +-
.../src/airflow_breeze/params/build_prod_params.py | 7 +-
.../airflow_breeze/params/common_build_params.py | 14 +-
.../src/airflow_breeze/params/shell_params.py | 1 +
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
dev/breeze/src/airflow_breeze/utils/cdxgen.py | 2 +-
.../src/airflow_breeze/utils/common_options.py | 9 +
.../src/airflow_breeze/utils/md5_build_check.py | 74 ++-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 2 +-
.../src/airflow_breeze/utils/selective_checks.py | 9 +-
dev/breeze/tests/test_selective_checks.py | 24 +-
dev/chart/build_changelog_annotations.py | 2 +-
dev/check_files.py | 4 +-
dev/perf/dags/elastic_dag.py | 2 +-
dev/prepare_release_issue.py | 2 +-
dev/send_email.py | 2 +-
dev/stats/get_important_pr_candidates.py | 11 +-
docker_tests/test_prod_image.py | 1 -
.../connections/aws.rst | 4 +-
.../connections/spark.rst | 2 +-
.../configurations-ref.rst | 0
.../index.rst | 3 +-
.../operators/cloud/bigquery_dts.rst | 2 +-
.../operators/cloud/kubernetes_engine.rst | 2 +-
.../operators/asb.rst | 2 +-
.../operators/qubole.rst | 4 +-
.../howto/create-custom-providers.rst | 2 +-
.../logging-monitoring/metrics.rst | 2 +-
.../modules_management.rst | 2 +-
docs/apache-airflow/best-practices.rst | 92 +++-
docs/apache-airflow/configurations-ref.rst | 1 +
docs/apache-airflow/core-concepts/operators.rst | 33 ++
docs/apache-airflow/extra-packages-ref.rst | 2 -
docs/apache-airflow/howto/docker-compose/index.rst | 2 +-
docs/apache-airflow/howto/email-config.rst | 2 +-
docs/apache-airflow/howto/operator/datetime.rst | 2 +-
docs/apache-airflow/howto/setup-and-teardown.rst | 8 +
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 8 +-
docs/apache-airflow/installation/prerequisites.rst | 2 +-
.../installation/supported-versions.rst | 2 +-
docs/apache-airflow/public-airflow-interface.rst | 6 +-
docs/apache-airflow/security/access-control.rst | 44 ++
.../security/secrets/secrets-backend/index.rst | 4 +-
docs/apache-airflow/security/security_model.rst | 10 +-
docs/apache-airflow/security/webserver.rst | 12 +-
docs/apache-airflow/tutorial/fundamentals.rst | 2 +-
docs/build_docs.py | 6 +-
docs/docker-stack/README.md | 10 +-
docs/docker-stack/build-arg-ref.rst | 1 +
docs/docker-stack/changelog.rst | 2 +-
.../customizing/own-requirements.sh | 2 +-
.../extending/add-airflow-configuration/Dockerfile | 2 +-
.../extending/add-apt-packages/Dockerfile | 2 +-
.../add-build-essential-extend/Dockerfile | 2 +-
.../extending/add-providers/Dockerfile | 2 +-
.../extending/add-pypi-packages/Dockerfile | 2 +-
.../extending/add-requirement-packages/Dockerfile | 2 +-
.../extending/custom-providers/Dockerfile | 2 +-
.../extending/embedding-dags/Dockerfile | 2 +-
.../extending/writable-directory/Dockerfile | 2 +-
docs/docker-stack/entrypoint.rst | 22 +-
docs/exts/docs_build/fetch_inventories.py | 4 +-
docs/exts/docs_build/lint_checks.py | 6 +-
generated/PYPI_README.md | 172 +++++++
generated/README.md | 6 +
generated/provider_dependencies.json | 31 +-
helm_tests/other/test_keda.py | 2 +-
images/breeze/output-commands-hash.txt | 28 +-
images/breeze/output_build-docs.svg | 46 +-
images/breeze/output_ci-image.svg | 24 +-
images/breeze/output_ci-image_build.svg | 174 ++++---
images/breeze/output_k8s.svg | 58 +--
images/breeze/output_k8s_deploy-airflow.svg | 68 +--
images/breeze/output_k8s_run-complete-tests.svg | 88 ++--
images/breeze/output_prod-image_build.svg | 186 +++----
images/breeze/output_release-management.svg | 6 +-
...tput_release-management_add-back-references.svg | 8 +-
...management_generate-issue-content-providers.svg | 42 +-
...elease-management_install-provider-packages.svg | 74 +--
...e-management_prepare-provider-documentation.svg | 46 +-
...elease-management_prepare-provider-packages.svg | 38 +-
.../output_release-management_publish-docs.svg | 110 ++--
...utput_release-management_update-constraints.svg | 40 +-
.../output_sbom_generate-provider-requirements.svg | 6 +-
images/breeze/output_setup_config.svg | 40 +-
images/breeze/output_static-checks.svg | 94 ++--
kubernetes_tests/test_kubernetes_pod_operator.py | 2 +-
scripts/ci/installed_providers.txt | 1 +
.../ci/license-templates/LICENSE.md | 2 +
.../ci/license-templates}/LICENSE.rst | 0
.../ci/license-templates}/LICENSE.txt | 0
scripts/ci/pre_commit/common_precommit_utils.py | 3 +-
.../pre_commit_check_deferrable_default.py | 2 +-
.../pre_commit_check_pre_commit_hooks.py | 5 +-
...it_check_providers_subpackages_all_have_init.py | 2 +-
.../pre_commit_checkout_no_credentials.py | 2 +-
.../pre_commit/pre_commit_generate_pypi_readme.py | 71 +++
scripts/ci/pre_commit/pre_commit_json_schema.py | 2 +-
.../pre_commit_sort_installed_providers.py | 3 +-
.../pre_commit_sort_spelling_wordlist.py | 3 +-
.../ci/pre_commit/pre_commit_supported_versions.py | 2 +-
.../pre_commit_update_common_sql_api_stubs.py | 24 +-
.../pre_commit_update_example_dags_paths.py | 5 +-
.../pre_commit_update_providers_dependencies.py | 38 +-
.../ci/pre_commit/pre_commit_version_heads_map.py | 74 ++-
.../docker/install_from_docker_context_files.sh | 45 +-
scripts/in_container/_in_container_utils.sh | 21 +-
scripts/in_container/run_generate_constraints.sh | 2 +-
scripts/in_container/run_migration_reference.py | 4 +-
.../in_container/run_provider_yaml_files_check.py | 21 +-
.../in_container/update_quarantined_test_status.py | 5 +-
scripts/in_container/verify_providers.py | 9 +-
setup.cfg | 24 +-
setup.py | 176 ++++---
tests/always/test_connection.py | 6 +-
tests/always/test_example_dags.py | 13 +-
tests/always/test_project_structure.py | 14 +-
.../endpoints/test_provider_endpoint.py | 53 +-
.../endpoints/test_variable_endpoint.py | 15 +
tests/auth/managers/fab/test_models.py | 62 +++
tests/cli/commands/test_config_command.py | 4 +-
tests/cli/commands/test_db_command.py | 2 +-
tests/conftest.py | 2 +-
tests/core/test_configuration.py | 22 +-
tests/core/test_policies.py | 4 +-
tests/dag_processing/test_job_runner.py | 35 +-
tests/decorators/test_python.py | 2 +-
tests/decorators/test_task_group.py | 46 ++
tests/jobs/test_scheduler_job.py | 28 +-
tests/jobs/test_triggerer_job_logging.py | 2 +-
tests/models/test_dag.py | 23 +-
tests/models/test_dagrun.py | 2 +-
tests/models/test_mappedoperator.py | 91 +++-
tests/models/test_param.py | 31 ++
tests/models/test_skipmixin.py | 2 +-
tests/models/test_taskinstance.py | 88 +++-
tests/models/test_taskmixin.py | 10 +-
tests/operators/test_python.py | 8 +-
.../amazon/aws/hooks/test_batch_waiters.py | 8 +-
tests/providers/amazon/aws/sensors/test_eks.py | 10 +-
.../amazon/aws/transfers/test_salesforce_to_s3.py | 21 +-
tests/providers/amazon/aws/utils/test_utils.py | 6 +-
tests/providers/apache/hive/hooks/test_hive.py | 29 +-
.../apache/hive/operators/test_hive_stats.py | 4 -
.../apache/hive/transfers/test_mssql_to_hive.py | 5 +-
.../apache/hive/transfers/test_mysql_to_hive.py | 16 +-
.../apache/hive/transfers/test_s3_to_hive.py | 11 +-
.../providers/apache/spark/hooks/test_spark_sql.py | 4 +-
tests/providers/apache/sqoop/hooks/test_sqoop.py | 51 +-
.../providers/apache/sqoop/operators/test_sqoop.py | 16 +-
.../providers/cncf/kubernetes/triggers/test_pod.py | 5 +-
tests/providers/common/sql/hooks/test_sql.py | 2 +-
.../databricks/hooks/test_databricks_sql.py | 2 +-
.../elasticsearch/log/elasticmock/__init__.py | 44 +-
.../log/elasticmock/fake_elasticsearch.py | 26 +-
.../log/elasticmock/utilities/__init__.py | 180 +++++++
.../elasticsearch/log/test_es_task_handler.py | 10 +-
tests/providers/exasol/hooks/test_sql.py | 2 +-
.../cloud/log/test_stackdriver_task_handler.py | 2 +-
.../test_cloud_storage_transfer_service.py | 9 +-
.../google/cloud/operators/test_workflows.py | 9 +-
.../google/cloud/transfers/test_postgres_to_gcs.py | 3 +-
.../cloud/transfers/test_salesforce_to_gcs.py | 18 +-
.../google/cloud/transfers/test_sql_to_gcs.py | 4 +-
.../test_cloud_storage_transfer_service.py | 2 +-
.../cloud/triggers/test_kubernetes_engine.py | 5 +-
tests/providers/grpc/operators/test_grpc.py | 4 +-
tests/providers/http/hooks/test_http.py | 9 +-
tests/providers/openlineage/plugins/test_utils.py | 6 +
tests/providers/snowflake/hooks/test_sql.py | 2 +-
tests/sensors/test_base.py | 64 ++-
tests/sensors/test_external_task_sensor.py | 8 +-
.../serialization/serializers/test_serializers.py | 4 +-
tests/serialization/test_dag_serialization.py | 9 +-
tests/serialization/test_pydantic_models.py | 14 +-
tests/system/conftest.py | 4 +-
tests/system/providers/amazon/CONTRIBUTING.md | 2 +-
tests/system/providers/amazon/aws/example_ec2.py | 3 +-
.../providers/amazon/aws/example_s3_to_sql.py | 2 +-
tests/system/providers/amazon/aws/utils/ec2.py | 2 +-
.../google/cloud/datapipelines}/__init__.py | 0
.../cloud/datapipelines/resources}/__init__.py | 0
.../cloud/sql_to_sheets/example_sql_to_sheets.py | 2 +-
.../providers/redis/example_redis_publish.py | 14 +-
tests/test_utils/mock_executor.py | 2 +-
tests/test_utils/providers.py | 2 +-
tests/test_utils/system_tests_class.py | 2 +-
tests/ti_deps/deps/test_trigger_rule_dep.py | 164 ++++++
tests/timetables/test_trigger_timetable.py | 18 +-
tests/utils/test_db_cleanup.py | 31 +-
tests/utils/test_helpers.py | 6 +-
tests/utils/test_sqlalchemy.py | 2 +-
tests/www/test_security.py | 134 +++++
tests/www/test_utils.py | 6 +-
tests/www/views/test_views_acl.py | 2 +-
tests/www/views/test_views_cluster_activity.py | 25 +-
tests/www/views/test_views_extra_links.py | 22 +-
tests/www/views/test_views_home.py | 6 +-
tests/www/views/test_views_log.py | 3 +-
tests/www/views/test_views_tasks.py | 3 +-
517 files changed, 5171 insertions(+), 3142 deletions(-)
copy docs/{apache-airflow-providers-apache-hive =>
apache-airflow-providers-elasticsearch}/configurations-ref.rst (100%)
create mode 100644 generated/PYPI_README.md
copy license-templates/LICENSE.txt => scripts/ci/license-templates/LICENSE.md
(98%)
rename {license-templates => scripts/ci/license-templates}/LICENSE.rst (100%)
rename {license-templates => scripts/ci/license-templates}/LICENSE.txt (100%)
create mode 100755 scripts/ci/pre_commit/pre_commit_generate_pypi_readme.py
create mode 100644 tests/auth/managers/fab/test_models.py
copy {airflow/api_connexion =>
tests/system/providers/google/cloud/datapipelines}/__init__.py (100%)
copy {airflow/api_connexion =>
tests/system/providers/google/cloud/datapipelines/resources}/__init__.py (100%)
