This is an automated email from the ASF dual-hosted git repository. mobuchowski pushed a commit to branch openlineage_dont_run_tis_executor in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9cf049c471df34b9d27eba073b0ae1545276382c Merge: 62aec10394 b8cdd284dd Author: Maciej Obuchowski <[email protected]> AuthorDate: Mon Aug 14 18:22:52 2023 +0200 Merge branch 'main' into openlineage_dont_run_tis_executor .github/actions/build-ci-images/action.yml | 7 - .github/actions/build-prod-images/action.yml | 4 - .github/workflows/build-images.yml | 1 + .github/workflows/ci.yml | 1 + .github/workflows/release_dockerhub_image.yml | 7 - BREEZE.rst | 23 +- Dockerfile | 14 +- Dockerfile.ci | 7 +- airflow/__main__.py | 1 - airflow/auth/managers/fab/models/__init__.py | 8 +- .../auth/managers/fab/security_manager/override.py | 53 +- airflow/config_templates/config.yml | 16 +- airflow/configuration.py | 70 +- airflow/models/trigger.py | 4 +- airflow/policies.py | 10 +- airflow/providers/amazon/aws/hooks/glue_catalog.py | 41 + .../amazon/aws/sensors/glue_catalog_partition.py | 32 +- airflow/providers/amazon/aws/triggers/glue.py | 83 + .../kubernetes/executors/kubernetes_executor.py | 3 +- airflow/providers/cncf/kubernetes/operators/pod.py | 4 +- .../providers/databricks/hooks/databricks_base.py | 148 +- airflow/providers/google/cloud/hooks/dataflow.py | 6 +- .../providers/google/cloud/hooks/datapipeline.py | 89 + airflow/providers/google/cloud/hooks/dataplex.py | 500 ++- .../google/cloud/operators/datapipeline.py | 102 + .../providers/google/cloud/operators/dataplex.py | 859 +++- airflow/providers/google/cloud/sensors/dataplex.py | 126 +- airflow/providers/google/provider.yaml | 11 + .../providers/microsoft/azure/secrets/key_vault.py | 9 + airflow/providers/openlineage/plugins/adapter.py | 4 +- airflow/providers/openlineage/plugins/listener.py | 21 +- airflow/secrets/cache.py | 18 +- airflow/secrets/local_filesystem.py | 6 +- airflow/security/permissions.py | 2 +- airflow/security/utils.py | 6 +- airflow/sensors/base.py | 10 +- airflow/sensors/date_time.py | 4 +- airflow/sensors/external_task.py | 50 +- airflow/sensors/time_delta.py | 2 +- airflow/sensors/time_sensor.py | 2 +- airflow/sentry.py | 4 +- airflow/serialization/serde.py | 4 +- airflow/serialization/serialized_objects.py | 20 +- airflow/settings.py | 43 +- airflow/task/task_runner/base_task_runner.py | 2 +- airflow/template/templater.py | 5 +- airflow/ti_deps/dep_context.py | 2 +- airflow/ti_deps/deps/base_ti_dep.py | 6 +- airflow/ti_deps/deps/dagrun_backfill_dep.py | 2 +- airflow/ti_deps/deps/pool_slots_available_dep.py | 2 +- airflow/ti_deps/deps/ready_to_reschedule.py | 2 +- airflow/timetables/interval.py | 3 +- airflow/triggers/base.py | 4 +- airflow/triggers/external_task.py | 18 +- airflow/triggers/file.py | 2 +- airflow/utils/db_cleanup.py | 4 + airflow/www/api/experimental/endpoints.py | 13 +- airflow/www/decorators.py | 3 +- airflow/www/extensions/init_appbuilder.py | 12 +- airflow/www/extensions/init_dagbag.py | 5 +- airflow/www/extensions/init_robots.py | 6 +- airflow/www/extensions/init_security.py | 5 +- airflow/www/fab_security/manager.py | 24 +- airflow/www/fab_security/sqla/manager.py | 11 +- airflow/www/forms.py | 5 +- airflow/www/security.py | 25 +- airflow/www/utils.py | 25 +- airflow/www/views.py | 48 +- chart/templates/flower/flower-ingress.yaml | 2 +- chart/templates/webserver/webserver-ingress.yaml | 2 +- chart/values.yaml | 2 + ...UALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md | 197 + .../src/airflow_breeze/commands/ci_commands.py | 10 + .../airflow_breeze/commands/ci_commands_config.py | 1 + .../airflow_breeze/commands/ci_image_commands.py | 187 +- .../commands/ci_image_commands_config.py | 10 +- .../commands/production_image_commands.py | 61 +- .../commands/production_image_commands_config.py | 3 +- .../commands/release_management_commands.py | 77 +- .../commands/release_management_commands_config.py | 14 +- dev/breeze/src/airflow_breeze/global_constants.py | 16 +- .../src/airflow_breeze/params/build_ci_params.py | 9 + .../airflow_breeze/params/common_build_params.py | 1 - .../src/airflow_breeze/utils/backtracking.py | 89 + .../src/airflow_breeze/utils/common_options.py | 28 +- .../airflow_breeze/utils/docker_command_utils.py | 11 - .../connections/databricks.rst | 10 +- .../operators/cloud/cloud_sql.rst | 26 +- .../cloud/cloud_storage_transfer_service.rst | 2 +- .../operators/cloud/datapipeline.rst | 60 + .../operators/cloud/dataplex.rst | 165 +- docs/apache-airflow/security/access-control.rst | 19 +- docs/apache-airflow/security/index.rst | 4 +- docs/apache-airflow/security/security_model.rst | 34 +- docs/spelling_wordlist.txt | 3 + helm_tests/other/test_keda.py | 2 +- helm_tests/webserver/test_ingress_flower.py | 34 + helm_tests/webserver/test_ingress_web.py | 33 + images/breeze/output-commands-hash.txt | 21 +- images/breeze/output_ci-image_build.svg | 124 +- images/breeze/output_ci.svg | 18 +- .../output_ci_find-backtracking-candidates.svg | 91 + images/breeze/output_prod-image_build.svg | 134 +- ...utput_release-management_update-constraints.svg | 70 +- .../output_setup_check-all-params-in-groups.svg | 36 +- .../output_setup_regenerate-command-images.svg | 62 +- kubernetes_tests/test_kubernetes_pod_operator.py | 22 +- .../pre_commit_check_pre_commit_hooks.py | 5 +- scripts/ci/pre_commit/pre_commit_json_schema.py | 2 +- .../pre_commit_update_common_sql_api_stubs.py | 22 +- .../pre_commit_update_example_dags_paths.py | 5 +- scripts/docker/clean-logs.sh | 2 + scripts/docker/install_additional_dependencies.sh | 2 +- scripts/docker/install_airflow.sh | 2 +- .../docker/install_from_docker_context_files.sh | 2 +- scripts/in_container/_in_container_utils.sh | 4 +- scripts/in_container/run_generate_constraints.sh | 20 + scripts/in_container/run_migration_reference.py | 4 +- .../in_container/update_quarantined_test_status.py | 5 +- scripts/in_container/verify_providers.py | 5 +- setup.py | 16 +- tests/auth/managers/fab/test_models.py | 62 + tests/core/test_policies.py | 4 +- tests/jobs/test_triggerer_job.py | 18 +- tests/models/test_trigger.py | 124 +- .../aws/sensors/test_glue_catalog_partition.py | 19 + tests/providers/amazon/aws/triggers/test_glue.py | 40 +- .../executors/test_kubernetes_executor.py | 13 + .../cncf/kubernetes/operators/test_pod.py | 6 +- .../providers/databricks/hooks/test_databricks.py | 123 +- .../google/cloud/hooks/test_datapipeline.py | 110 + .../providers/google/cloud/hooks/test_dataplex.py | 155 +- .../google/cloud/operators/test_datapipeline.py | 138 + .../google/cloud/operators/test_dataplex.py | 274 ++ .../google/cloud/sensors/test_dataplex.py | 47 +- .../google/cloud/cloud_sql/example_cloud_sql.py | 49 +- .../cloud_sql/example_cloud_sql_deferrable.py | 184 - .../cloud_sql/example_cloud_sql_query_mysql.py | 280 ++ .../cloud_sql/example_cloud_sql_query_postgres.py | 288 ++ .../google/cloud/datapipelines/__init__.py | 17 - .../cloud/datapipelines/example_datapipeline.py | 124 + .../cloud/datapipelines/resources/__init__.py | 17 - .../cloud/datapipelines/resources/kinglear.txt | 4273 ++++++++++++++++++++ .../cloud/datapipelines/resources/word-count.json | 29 + .../google/cloud/dataplex/example_dataplex_dq.py | 343 ++ .../example_cloud_storage_transfer_service_gcp.py | 42 +- ...e_cloud_storage_transfer_service_gcs_to_gcs.py} | 20 +- .../resources/{file => transfer_service_gcp_file} | 0 .../{file => transfer_service_gcs_to_gcs_file} | 0 tests/www/views/test_views_custom_user_views.py | 112 + tests/www/views/test_views_home.py | 52 +- 151 files changed, 10297 insertions(+), 1121 deletions(-)
