This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from ffa33e4 (commit)
to a0374a5 (commit)
from ffa33e4 Task Instance Modal UX Enhancements (#10944)
add e066260 Improve the Error message in Breeze for invalid params
(#10980)
add 284c373 Remove redundant parentheses in serialized_objects.py (#10966)
add a575c79 Fix 'Upload documentation' step in CI (#10981)
add 6e5cc4c Pins temporarily moto to 1.3.14 (#10986)
add 4a46f43 Allows to build production images for 1.10.2 and 1.10.1
Airflow (#10983)
add 82a9477 The test_find_not_should_ignore_path is now in heisentests
(#10989)
add f934e2f Unpin 'tornado' dep pulled in by flower (#10993)
add cba51d4 Simplify the K8sExecutor and K8sPodOperator (#10393)
add 5b3fb53 Add new teammate to Polidea (#11000)
add 966a06d Fetching databricks host from connection if not supplied in
extras. (#10762)
add 1a46e9b Remove redundant curly brace from breeze echo message (#11012)
add 1539bd0 KubernetesJobWatcher no longer inherits from Process (#11017)
add f16f474 Replace PNG/text with SVG that includes name in proper
typography (#11018)
add 49c193f [AIP-34] TaskGroup: A UI task grouping concept as an
alternative to SubDagOperator (#10153)
add 9edfcb7 Support extra_args in S3Hook and GCSToS3Operator (#11001)
add 78c3424 Remove Edit Button from DagModel View (#11026)
add 9dd8310 Increase typing coverage JDBC provider (#11021)
add 2bf7b7c Add typing to amazon provider EMR (#10910)
add 980aa39 Fix typo in
DagFileProcessorAgent._last_parsing_stat_received_at (#11022)
add d7f1666 Fix logo issues due to generic scripting selector use (#11028)
add 2410f59 Get Airflow configs with sensitive data from AWS Systems
Manager (#11023)
add cac5577 Refactor rebase copy (#11030)
add b61225a Add D204 pydocstyle check (#11031)
add 17faea0 Starting breeze will run an init script after the environment
is setup (#11029)
add 19d781f Replace JS package toggle w/ pure CSS solution (#11035)
add 7e112b1 Only gather KinD logs if tests fail (#11058)
add 34c06d7 Separates out user documentation for production images.
(#10998)
add 3db4d3b All versions in CI yamls are not hard-coded any more (#10959)
add c362d69 Add Workflow to delete old artifacts (#11064)
add 2afb20d [Doc] Correct description for macro task_instance_key_str
(#11062)
add f4513c0 Revert "KubernetesJobWatcher no longer inherits from Process
(#11017)" (#11065)
add a449b1f Add JSON schema validation for Helm values (#10664)
add cb979f9 Get Airflow configs with sensitive data from
CloudSecretManagerBackend (#11024)
add 45639cd Add some tasks using BashOperator in TaskGroup example dag
(#11072)
add e3a5900 Replace Airflow Slack Invite old link to short link (#11071)
add 29d6297 Fix s.apache.org Slack link (#11078)
add 1ebd3a6 Pandas behaviour for None changed in 1.1.2 (#11004)
add cea9e82 Improves deletion of old artifacts. (#11079)
add 52fdb62 Requirements might get upgraded without setup.py change
(#10784)
add f3e87c5 Add D202 pydocstyle check (#11032)
add fbd994a Add permissions for stable API (#10594)
add e59ad5b Make Skipmixin handle empty branch properly (#10751)
add 423a382 SkipMixin: Add missing session.commit() and test (#10421)
add 1a14982 Fix typo in STATIC_CODE_CHECKS.rst (#11094)
add 35c4398 Avoid redundant SET conversion (#11091)
add a0374a5 Fix for pydocstyle D202 (#11096)
No new revisions were added by this update.
Summary of changes:
.github/boring-cyborg.yml | 2 +-
.github/workflows/build-images-workflow-run.yml | 40 +-
.github/workflows/ci.yml | 205 ++--
.github/workflows/delete_old_artifacts.yml | 14 +
.pre-commit-config.yaml | 2 +-
.rat-excludes | 2 -
BREEZE.rst | 13 +-
CI.rst | 17 +-
CONTRIBUTING.rst | 109 +-
Dockerfile | 3 +
Dockerfile.ci | 2 +-
IMAGES.rst | 298 +-----
INTHEWILD.md | 2 +-
README.md | 38 +-
STATIC_CODE_CHECKS.rst | 2 +-
UPDATING.md | 522 +++++++++-
airflow/api/auth/backend/kerberos_auth.py | 2 +-
.../api/common/experimental/get_dag_run_state.py | 1 -
airflow/api_connexion/endpoints/config_endpoint.py | 2 +-
.../api_connexion/endpoints/connection_endpoint.py | 10 +-
airflow/api_connexion/endpoints/dag_endpoint.py | 20 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 34 +-
.../api_connexion/endpoints/dag_source_endpoint.py | 2 +-
.../api_connexion/endpoints/event_log_endpoint.py | 5 +-
.../api_connexion/endpoints/extra_link_endpoint.py | 9 +-
.../endpoints/import_error_endpoint.py | 5 +-
airflow/api_connexion/endpoints/log_endpoint.py | 2 +-
airflow/api_connexion/endpoints/pool_endpoint.py | 11 +-
airflow/api_connexion/endpoints/task_endpoint.py | 4 +-
.../endpoints/task_instance_endpoint.py | 15 +-
.../api_connexion/endpoints/variable_endpoint.py | 10 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 26 +-
airflow/api_connexion/schemas/common_schema.py | 2 -
airflow/api_connexion/schemas/dag_schema.py | 1 -
airflow/api_connexion/security.py | 78 +-
airflow/cli/cli_parser.py | 4 +
airflow/cli/commands/dag_command.py | 4 +-
airflow/cli/commands/task_command.py | 2 -
airflow/cli/commands/webserver_command.py | 1 +
airflow/config_templates/config.yml | 431 +-------
airflow/config_templates/default_airflow.cfg | 267 +----
airflow/configuration.py | 1 -
airflow/contrib/hooks/aws_hook.py | 1 +
airflow/contrib/hooks/gcp_api_base_hook.py | 1 +
airflow/contrib/hooks/gcp_dataflow_hook.py | 1 +
airflow/contrib/hooks/gcs_hook.py | 1 +
airflow/contrib/operators/dataflow_operator.py | 3 +
airflow/contrib/operators/gcp_compute_operator.py | 6 +
airflow/contrib/operators/gcs_to_s3.py | 1 +
.../contrib/operators/postgres_to_gcs_operator.py | 1 +
airflow/contrib/sensors/gcp_transfer_sensor.py | 1 +
airflow/dag/base_dag.py | 1 +
.../example_kubernetes_executor_config.py | 18 +-
airflow/example_dags/example_task_group.py | 58 ++
airflow/exceptions.py | 7 +
airflow/executors/base_executor.py | 1 +
airflow/executors/celery_executor.py | 3 +-
airflow/executors/dask_executor.py | 1 +
airflow/executors/kubernetes_executor.py | 191 +---
airflow/executors/local_executor.py | 6 +
airflow/hooks/dbapi_hook.py | 4 +-
airflow/jobs/backfill_job.py | 3 +-
airflow/jobs/local_task_job.py | 1 -
airflow/jobs/scheduler_job.py | 1 -
airflow/kubernetes/k8s_model.py | 1 +
airflow/kubernetes/kube_client.py | 1 -
airflow/kubernetes/pod.py | 127 ---
airflow/kubernetes/pod_generator.py | 295 ++----
airflow/kubernetes/pod_generator_deprecated.py | 343 +++++++
airflow/kubernetes/pod_launcher.py | 2 +
airflow/kubernetes/refresh_config.py | 2 +
airflow/kubernetes/volume.py | 56 --
airflow/kubernetes/volume_mount.py | 72 --
airflow/kubernetes/worker_configuration.py | 448 ---------
.../basic_template.yaml} | 91 +-
airflow/lineage/__init__.py | 1 +
airflow/lineage/entities.py | 5 +
airflow/macros/__init__.py | 1 -
.../versions/3c20cacc0044_add_dagrun_run_type.py | 1 +
...a59344a4_make_taskinstance_pool_not_nullable.py | 1 -
.../8f966b9c467a_set_conn_type_as_non_nullable.py | 1 -
airflow/models/baseoperator.py | 39 +-
airflow/models/connection.py | 1 +
airflow/models/crypto.py | 2 +
airflow/models/dag.py | 55 +-
airflow/models/dagbag.py | 2 +-
airflow/models/dagcode.py | 1 +
airflow/models/dagrun.py | 3 +-
airflow/models/errors.py | 1 +
airflow/models/kubernetes.py | 2 +
airflow/models/pool.py | 2 +
airflow/models/serialized_dag.py | 1 +
airflow/models/skipmixin.py | 33 +-
airflow/models/slamiss.py | 1 +
airflow/models/taskinstance.py | 9 +-
airflow/models/taskmixin.py | 11 +
airflow/models/variable.py | 2 +-
airflow/models/xcom.py | 1 +
airflow/models/xcom_arg.py | 5 +
airflow/mypy/plugin/decorators.py | 1 +
airflow/operators/bash.py | 1 +
airflow/operators/python.py | 1 +
airflow/operators/sql.py | 1 -
airflow/operators/subdag_operator.py | 1 +
airflow/plugins_manager.py | 1 +
.../providers/amazon/aws/hooks/cloud_formation.py | 3 -
airflow/providers/amazon/aws/hooks/emr.py | 6 +-
airflow/providers/amazon/aws/hooks/glue_catalog.py | 2 -
airflow/providers/amazon/aws/hooks/kinesis.py | 1 -
.../providers/amazon/aws/hooks/lambda_function.py | 1 -
airflow/providers/amazon/aws/hooks/logs.py | 1 -
airflow/providers/amazon/aws/hooks/s3.py | 21 +-
airflow/providers/amazon/aws/hooks/sagemaker.py | 20 -
airflow/providers/amazon/aws/operators/datasync.py | 1 +
.../amazon/aws/operators/emr_add_steps.py | 17 +-
.../amazon/aws/operators/emr_create_job_flow.py | 19 +-
.../amazon/aws/operators/emr_modify_cluster.py | 5 +-
.../amazon/aws/operators/emr_terminate_job_flow.py | 7 +-
airflow/providers/amazon/aws/operators/sqs.py | 1 -
.../amazon/aws/secrets/systems_manager.py | 13 +
.../providers/amazon/aws/transfers/gcs_to_s3.py | 8 +-
.../providers/amazon/aws/transfers/s3_to_sftp.py | 1 -
.../providers/amazon/aws/transfers/sftp_to_s3.py | 1 -
.../apache/druid/transfers/hive_to_druid.py | 1 -
airflow/providers/apache/hive/hooks/hive.py | 1 -
airflow/providers/apache/pig/hooks/pig.py | 1 -
.../apache/spark/hooks/spark_jdbc_script.py | 2 -
.../providers/apache/spark/hooks/spark_submit.py | 3 -
.../kubernetes/example_dags/example_kubernetes.py | 28 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 1 -
.../cncf/kubernetes/operators/kubernetes_pod.py | 138 ++-
airflow/providers/databricks/hooks/databricks.py | 5 +-
.../providers/elasticsearch/log/es_task_handler.py | 1 -
airflow/providers/google/cloud/hooks/bigquery.py | 4 -
airflow/providers/google/cloud/hooks/bigtable.py | 2 -
.../cloud/hooks/cloud_storage_transfer_service.py | 1 -
airflow/providers/google/cloud/hooks/dataprep.py | 3 -
airflow/providers/google/cloud/hooks/dlp.py | 30 -
airflow/providers/google/cloud/hooks/gcs.py | 5 -
.../google/cloud/hooks/kubernetes_engine.py | 2 -
.../providers/google/cloud/hooks/stackdriver.py | 6 -
airflow/providers/google/cloud/hooks/tasks.py | 13 -
.../operators/cloud_storage_transfer_service.py | 1 -
.../providers/google/cloud/operators/mlengine.py | 1 -
airflow/providers/google/cloud/operators/pubsub.py | 1 -
.../google/cloud/secrets/secret_manager.py | 14 +
airflow/providers/google/cloud/sensors/pubsub.py | 1 -
.../google/marketing_platform/hooks/analytics.py | 5 -
.../marketing_platform/hooks/display_video.py | 6 -
airflow/providers/imap/hooks/imap.py | 1 -
airflow/providers/jdbc/hooks/jdbc.py | 24 +-
.../providers/microsoft/azure/hooks/azure_batch.py | 2 -
airflow/providers/microsoft/azure/hooks/wasb.py | 1 -
airflow/providers/mysql/hooks/mysql.py | 1 -
airflow/providers/redis/operators/redis_publish.py | 1 -
airflow/providers/salesforce/hooks/salesforce.py | 1 -
airflow/providers/sendgrid/utils/emailer.py | 1 -
airflow/providers/slack/hooks/slack.py | 1 -
airflow/providers/ssh/hooks/ssh.py | 2 -
airflow/sensors/base_sensor_operator.py | 1 +
airflow/sensors/external_task_sensor.py | 2 +
airflow/sensors/filesystem.py | 1 +
airflow/sensors/smart_sensor_operator.py | 4 +
airflow/serialization/enums.py | 3 +
airflow/serialization/schema.json | 48 +-
airflow/serialization/serialized_objects.py | 99 +-
airflow/settings.py | 1 -
airflow/stats.py | 2 +
airflow/task/task_runner/standard_task_runner.py | 1 +
airflow/ti_deps/dep_context.py | 1 +
airflow/ti_deps/deps/base_ti_dep.py | 1 +
airflow/ti_deps/deps/dag_ti_slots_available_dep.py | 1 +
airflow/ti_deps/deps/dag_unpaused_dep.py | 1 +
airflow/ti_deps/deps/dagrun_exists_dep.py | 1 +
airflow/ti_deps/deps/dagrun_id_dep.py | 1 +
.../ti_deps/deps/exec_date_after_start_date_dep.py | 1 +
airflow/ti_deps/deps/not_in_retry_period_dep.py | 1 +
airflow/ti_deps/deps/pool_slots_available_dep.py | 1 +
airflow/ti_deps/deps/prev_dagrun_dep.py | 1 +
airflow/ti_deps/deps/ready_to_reschedule.py | 1 +
airflow/ti_deps/deps/runnable_exec_date_dep.py | 1 +
airflow/ti_deps/deps/task_concurrency_dep.py | 1 +
airflow/ti_deps/deps/task_not_running_dep.py | 1 +
airflow/ti_deps/deps/trigger_rule_dep.py | 2 +-
airflow/ti_deps/deps/valid_state_dep.py | 1 +
airflow/utils/cli.py | 2 +-
airflow/utils/dag_processing.py | 15 +-
airflow/utils/dates.py | 1 -
airflow/utils/db.py | 1 -
airflow/utils/decorators.py | 1 -
airflow/utils/email.py | 1 -
airflow/utils/file.py | 3 -
airflow/utils/json.py | 9 +
airflow/utils/log/colored_log.py | 1 +
airflow/utils/log/file_task_handler.py | 1 +
airflow/utils/log/json_formatter.py | 1 +
airflow/utils/log/log_reader.py | 5 -
airflow/utils/log/logging_mixin.py | 4 +
.../log/task_handler_with_custom_formatter.py | 1 +
airflow/utils/mixins.py | 1 +
airflow/utils/operator_resources.py | 6 +
airflow/utils/sqlalchemy.py | 1 +
airflow/utils/state.py | 1 +
airflow/utils/task_group.py | 379 +++++++
airflow/utils/timezone.py | 2 -
airflow/utils/trigger_rule.py | 1 +
airflow/utils/types.py | 1 +
airflow/utils/weekday.py | 1 +
airflow/utils/weight_rule.py | 1 +
airflow/www/api/experimental/endpoints.py | 5 -
airflow/www/extensions/init_appbuilder.py | 1 +
airflow/www/forms.py | 4 +
airflow/www/package.json | 1 -
airflow/www/security.py | 177 ++--
airflow/www/static/css/graph.css | 14 +
airflow/www/static/css/main.css | 8 +-
airflow/www/static/css/switch.css | 84 ++
airflow/www/templates/airflow/circles.html | 4 +-
airflow/www/templates/airflow/dag.html | 12 +-
airflow/www/templates/airflow/dags.html | 11 +-
airflow/www/templates/airflow/graph.html | 462 +++++++--
airflow/www/templates/airflow/tree.html | 8 +-
airflow/www/templates/appbuilder/navbar.html | 56 +-
airflow/www/utils.py | 3 +-
airflow/www/validators.py | 1 +
airflow/www/views.py | 207 +++-
airflow/www/webpack.config.js | 4 +-
airflow/www/widgets.py | 2 +
airflow/www/yarn.lock | 5 -
breeze | 13 +-
breeze-complete | 25 +-
chart/values.schema.json | 1055 ++++++++++++++++++++
dev/remove_artifacts.sh | 84 ++
docs/build_docs.py | 4 +-
docs/concepts.rst | 42 +
docs/howto/operator/kubernetes.rst | 20 +-
docs/img/task_group.gif | Bin 0 -> 609981 bytes
docs/index.rst | 1 +
docs/macros-ref.rst | 2 +-
docs/production-deployment.rst | 457 +++++++++
docs/project.rst | 2 +-
docs/security/access-control.rst | 78 +-
.../google-cloud-secret-manager-backend.rst | 2 +
docs/security/secrets/secrets-backend/index.rst | 3 +
docs/spelling_wordlist.txt | 2 +
kubernetes_tests/test_kubernetes_pod_operator.py | 177 ++--
licenses/LICENSE-bootstrap-toggle.txt | 21 -
metastore_browser/hive_metastore.py | 1 +
scripts/ci/docker-compose/base.yml | 1 +
scripts/ci/libraries/_build_images.sh | 5 +
scripts/ci/libraries/_initialization.sh | 89 +-
scripts/ci/libraries/_kind.sh | 2 +
scripts/ci/libraries/_parameters.sh | 2 +-
scripts/in_container/check_environment.sh | 2 +
scripts/in_container/entrypoint_ci.sh | 3 +
scripts/in_container/entrypoint_exec.sh | 3 +
scripts/in_container/run_docs_build.sh | 2 +-
scripts/in_container/run_init_script.sh | 45 +
setup.cfg | 1 -
setup.py | 4 +-
.../endpoints/test_config_endpoint.py | 23 +-
.../endpoints/test_connection_endpoint.py | 21 +-
tests/api_connexion/endpoints/test_dag_endpoint.py | 86 +-
.../endpoints/test_dag_run_endpoint.py | 135 ++-
.../endpoints/test_dag_source_endpoint.py | 20 +-
.../endpoints/test_event_log_endpoint.py | 13 +-
.../endpoints/test_extra_link_endpoint.py | 22 +-
.../endpoints/test_import_error_endpoint.py | 16 +-
tests/api_connexion/endpoints/test_log_endpoint.py | 24 +-
.../api_connexion/endpoints/test_pool_endpoint.py | 20 +-
.../api_connexion/endpoints/test_task_endpoint.py | 19 +-
.../endpoints/test_task_instance_endpoint.py | 2 +-
.../endpoints/test_variable_endpoint.py | 24 +-
.../endpoints/test_version_endpoint.py | 4 +-
.../api_connexion/endpoints/test_xcom_endpoint.py | 153 ++-
tests/bats/test_breeze_complete.bats | 105 ++
tests/bats/test_breeze_params.bats | 6 +-
tests/executors/test_kubernetes_executor.py | 104 +-
tests/kubernetes/models/test_pod.py | 77 --
tests/kubernetes/models/test_secret.py | 84 +-
.../{pod.yaml => pod_generator_base.yaml} | 31 +-
...d.yaml => pod_generator_base_with_secrets.yaml} | 40 +-
tests/kubernetes/test_pod_generator.py | 682 +++++--------
tests/kubernetes/test_worker_configuration.py | 930 -----------------
tests/models/test_skipmixin.py | 28 +
tests/operators/test_python.py | 39 +
tests/plugins/test_plugin_ignore.py | 4 +
tests/providers/amazon/aws/hooks/test_s3.py | 21 +
.../amazon/aws/secrets/test_systems_manager.py | 14 +
.../amazon/aws/transfers/test_gcs_to_s3.py | 51 +
.../kubernetes/operators/test_kubernetes_pod.py | 2 +-
.../providers/databricks/hooks/test_databricks.py | 11 +
.../google/cloud/secrets/test_secret_manager.py | 20 +
.../providers/salesforce/hooks/test_salesforce.py | 8 +-
tests/serialization/test_dag_serialization.py | 63 ++
tests/test_config_templates.py | 6 +-
tests/test_utils/api_connexion_utils.py | 54 +-
tests/utils/test_task_group.py | 561 +++++++++++
tests/www/test_views.py | 15 +-
299 files changed, 7241 insertions(+), 4728 deletions(-)
create mode 100644 .github/workflows/delete_old_artifacts.yml
create mode 100644 airflow/example_dags/example_task_group.py
delete mode 100644 airflow/kubernetes/pod.py
create mode 100644 airflow/kubernetes/pod_generator_deprecated.py
delete mode 100644 airflow/kubernetes/volume.py
delete mode 100644 airflow/kubernetes/volume_mount.py
delete mode 100644 airflow/kubernetes/worker_configuration.py
copy
airflow/{kubernetes/pod_template_file_examples/dags_in_image_template.yaml =>
kubernetes_executor_templates/basic_template.yaml} (51%)
create mode 100644 airflow/utils/task_group.py
create mode 100644 airflow/www/static/css/switch.css
create mode 100644 chart/values.schema.json
create mode 100755 dev/remove_artifacts.sh
create mode 100644 docs/img/task_group.gif
create mode 100644 docs/production-deployment.rst
delete mode 100644 licenses/LICENSE-bootstrap-toggle.txt
create mode 100755 scripts/in_container/run_init_script.sh
delete mode 100644 tests/kubernetes/models/test_pod.py
copy tests/kubernetes/{pod.yaml => pod_generator_base.yaml} (63%)
copy tests/kubernetes/{pod.yaml => pod_generator_base_with_secrets.yaml} (54%)
delete mode 100644 tests/kubernetes/test_worker_configuration.py
create mode 100644 tests/utils/test_task_group.py
