This is an automated email from the ASF dual-hosted git repository.
dheerajturaga pushed a change to branch aip-125/20-demo-kit
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit a87c71cc2e4 [DO NOT MERGE] AIP-125 proof of concept: demo kit
add 70f1bb6cf1c AIP-125: Move the webhook match grammar into the shared
library
add 2c25bed13f3 AIP-125: Add WebhookMatch and cancel_on to the Task SDK
add f7ffdfbb00c AIP-125: Serialize cancel_on and store Dag cancel
conditions
add 47c0d720ed0 AIP-125: Record cancel requests from webhook deliveries
add 099b65909ee AIP-125: Apply webhook cancel requests in the scheduler
add 2b9f0d0be3d AIP-125: Show cancel conditions and cancel requests
add 6c91a8820fd AIP-125: Document cancelling runs and tasks from webhooks
add 70470266b22 [DO NOT MERGE] AIP-125 proof of concept: demo kit
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a87c71cc2e4)
\
N -- N -- N refs/heads/aip-125/20-demo-kit (70470266b22)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../logging-monitoring/callbacks.rst | 7 +
.../administration-and-deployment/webhooks.rst | 332 ++++--
.../authoring-and-scheduling/asset-scheduling.rst | 3 +-
.../docs/authoring-and-scheduling/index.rst | 1 +
.../authoring-and-scheduling/webhook-cancel.rst | 442 ++++++++
.../authoring-and-scheduling/webhook-topics.rst | 29 +-
.../authoring-and-scheduling/webhook-waits.rst | 64 +-
airflow-core/docs/howto/usage-cli.rst | 5 +
airflow-core/docs/migrations-ref.rst | 6 +-
airflow-core/docs/public-airflow-interface.rst | 33 +-
airflow-core/docs/security/audit_logs.rst | 8 +-
airflow-core/docs/security/security_model.rst | 20 +-
airflow-core/newsfragments/75577.significant.rst | 9 +
airflow-core/src/airflow/api/common/mark_tasks.py | 22 +-
.../api_fastapi/core_api/datamodels/dags.py | 15 +
.../api_fastapi/core_api/datamodels/tasks.py | 33 +-
.../api_fastapi/core_api/datamodels/webhooks.py | 158 ++-
.../core_api/openapi/v2-rest-api-generated.yaml | 296 ++++-
.../api_fastapi/core_api/routes/public/webhooks.py | 36 +-
.../execution_api/datamodels/inbound_wait.py | 4 +-
airflow-core/src/airflow/cli/cli_config.py | 13 +-
.../src/airflow/cli/commands/webhooks_command.py | 152 ++-
.../src/airflow/config_templates/config.yml | 49 +-
.../src/airflow/dag_processing/collection.py | 116 +-
.../src/airflow/jobs/scheduler_job_runner.py | 154 ++-
.../0151_3_6_0_add_dag_cancel_condition_table.py | 72 ++
.../0152_3_6_0_add_webhook_cancel_request_table.py | 92 ++
airflow-core/src/airflow/models/__init__.py | 2 +
airflow-core/src/airflow/models/dag.py | 2 +
.../src/airflow/models/dag_cancel_condition.py | 64 ++
.../src/airflow/models/inbound_delivery.py | 21 +-
.../src/airflow/models/task_coordinates.py | 21 +-
.../src/airflow/models/webhook_cancel_request.py | 98 ++
.../src/airflow/serialization/dag_version_diff.py | 9 +-
airflow-core/src/airflow/serialization/decoders.py | 45 +
.../serialization/definitions/baseoperator.py | 4 +
.../src/airflow/serialization/definitions/dag.py | 14 +-
.../serialization/definitions/mappedoperator.py | 9 +
.../serialization/definitions/webhook_match.py | 106 ++
airflow-core/src/airflow/serialization/encoders.py | 53 +-
airflow-core/src/airflow/serialization/schema.json | 31 +-
.../airflow/serialization/serialized_objects.py | 77 +-
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 356 +++++-
.../airflow/ui/openapi-gen/requests/types.gen.ts | 163 ++-
.../airflow/ui/public/i18n/locales/en/browse.json | 42 +-
.../airflow/ui/public/i18n/locales/en/common.json | 20 +-
.../ui/src/components/WebhookMatchList.test.tsx | 78 ++
.../airflow/ui/src/components/WebhookMatchList.tsx | 87 ++
.../src/airflow/ui/src/constants/stateOptions.ts | 5 +
.../src/airflow/ui/src/pages/Dag/Details.test.tsx | 112 ++
.../src/airflow/ui/src/pages/Dag/Details.tsx | 16 +-
.../pages/TaskInstance/CancelConditions.test.tsx | 117 ++
.../ui/src/pages/TaskInstance/CancelConditions.tsx | 64 ++
.../ui/src/pages/TaskInstance/Details.test.tsx | 26 +-
.../airflow/ui/src/pages/TaskInstance/Details.tsx | 2 +
.../ui/src/pages/TaskInstance/WebhookWait.tsx | 9 +-
.../WebhookDeliveries/DeliveryDetails.test.tsx | 264 ++++-
.../pages/WebhookDeliveries/DeliveryDetails.tsx | 217 +++-
.../WebhookDeliveries/WebhookDeliveries.test.tsx | 20 +-
.../ui/src/pages/WebhookDeliveries/outcomes.ts | 53 +
.../{rules/levels.js => src/utils/webhookMatch.ts} | 14 +-
airflow-core/src/airflow/utils/db.py | 1 +
airflow-core/src/airflow/utils/db_cleanup.py | 4 +
.../src/airflow/webhooks/cancel_application.py | 646 +++++++++++
.../src/airflow/webhooks/cancel_requests.py | 251 +++++
airflow-core/src/airflow/webhooks/pipeline.py | 135 ++-
airflow-core/src/airflow/webhooks/registry.py | 5 +
airflow-core/src/airflow/webhooks/wait_matching.py | 29 +-
.../src/airflow/webhooks/wait_registration.py | 51 +-
.../src/airflow/webhooks/wait_resumption.py | 19 +-
airflow-core/src/airflow/webhooks/waits.py | 163 +--
.../core_api/routes/public/test_dags.py | 31 +
.../core_api/routes/public/test_tasks.py | 88 +-
.../core_api/routes/public/test_webhooks.py | 239 +++-
.../versions/head/test_inbound_waits.py | 22 +-
.../versions/head/test_task_instances.py | 2 +-
.../tests/unit/api_fastapi/webhooks/test_app.py | 23 +-
.../unit/cli/commands/test_webhooks_command.py | 234 +++-
.../tests/unit/dag_processing/test_collection.py | 142 ++-
.../tests/unit/dag_processing/test_manager.py | 5 +-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 687 +++++++++++-
.../tests/unit/models/test_dag_cancel_condition.py | 85 ++
.../unit/models/test_webhook_cancel_request.py | 109 ++
.../definitions/test_webhook_match.py | 121 ++
.../unit/serialization/test_dag_serialization.py | 206 ++++
.../unit/serialization/test_dag_version_diff.py | 65 +-
.../tests/unit/serialization/test_decoders.py | 48 +-
.../tests/unit/serialization/test_encoders.py | 44 +-
airflow-core/tests/unit/utils/test_db_cleanup.py | 68 ++
.../tests/unit/webhooks/test_cancel_application.py | 1171 ++++++++++++++++++++
.../tests/unit/webhooks/test_cancel_requests.py | 681 ++++++++++++
.../unit/webhooks/test_pipeline_concurrency.py | 88 +-
airflow-core/tests/unit/webhooks/test_registry.py | 13 +
airflow-core/tests/unit/webhooks/test_topics.py | 21 +-
.../tests/unit/webhooks/test_wait_matching.py | 123 +-
.../tests/unit/webhooks/test_wait_registration.py | 2 +-
.../tests/unit/webhooks/test_wait_resumption.py | 36 +-
airflow-core/tests/unit/webhooks/test_waits.py | 216 +---
.../src/airflowctl/api/datamodels/generated.py | 283 ++++-
dev/aip125_demo/PRESENTER.md | 140 ++-
dev/aip125_demo/README.md | 232 +++-
dev/aip125_demo/bench_dags/cancel_bench.py | 58 +
dev/aip125_demo/bench_latency.py | 34 +-
dev/aip125_demo/dags/merge_gate_operator.py | 7 +-
dev/aip125_demo/dags/preview_environment.py | 109 ++
dev/aip125_demo/dags/preview_with_teardown.py | 68 ++
dev/aip125_demo/dags/ship_when_labeled.py | 73 ++
dev/aip125_demo/demo.sh | 573 +++++++++-
...d.json => github_pull_request_synchronize.json} | 7 +-
devel-common/src/tests_common/test_utils/db.py | 8 +
.../src/tests_common/test_utils/webhooks.py | 69 +-
go-sdk/internal/genspec/authoring.go | 3 +
go-sdk/schema/dag-schema.json | 31 +-
java-sdk/sdk/build.gradle.kts | 2 +
java-sdk/sdk/schema/dag-schema.json | 31 +-
providers/github/docs/webhooks.rst | 20 +-
.../standard/docs/connections/inbound_webhook.rst | 12 +-
providers/standard/docs/operators/webhooks.rst | 13 +-
.../observability/metrics/metrics_template.yaml | 9 +
.../src/airflow_shared/webhook_naming/matching.py | 158 +--
.../tests/webhook_naming/test_matching.py | 114 +-
task-sdk/docs/api.rst | 4 +
task-sdk/src/airflow/sdk/__init__.py | 3 +
task-sdk/src/airflow/sdk/__init__.pyi | 2 +
task-sdk/src/airflow/sdk/bases/decorator.py | 3 +
task-sdk/src/airflow/sdk/bases/operator.py | 35 +
task-sdk/src/airflow/sdk/definitions/dag.py | 29 +
.../src/airflow/sdk/definitions/mappedoperator.py | 28 +
task-sdk/src/airflow/sdk/definitions/webhooks.py | 153 +++
task-sdk/tests/task_sdk/bases/test_decorator.py | 58 +-
task-sdk/tests/task_sdk/bases/test_operator.py | 89 +-
task-sdk/tests/task_sdk/definitions/test_dag.py | 133 +++
.../task_sdk/definitions/test_mappedoperator.py | 35 +-
.../tests/task_sdk/definitions/test_webhooks.py | 175 +++
ts-sdk/schema/dag-schema.json | 31 +-
ts-sdk/scripts/generate-dag-schema.mjs | 3 +
136 files changed, 11859 insertions(+), 1242 deletions(-)
create mode 100644
airflow-core/docs/authoring-and-scheduling/webhook-cancel.rst
create mode 100644 airflow-core/newsfragments/75577.significant.rst
create mode 100644
airflow-core/src/airflow/migrations/versions/0151_3_6_0_add_dag_cancel_condition_table.py
create mode 100644
airflow-core/src/airflow/migrations/versions/0152_3_6_0_add_webhook_cancel_request_table.py
create mode 100644 airflow-core/src/airflow/models/dag_cancel_condition.py
create mode 100644 airflow-core/src/airflow/models/webhook_cancel_request.py
create mode 100644
airflow-core/src/airflow/serialization/definitions/webhook_match.py
create mode 100644
airflow-core/src/airflow/ui/src/components/WebhookMatchList.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/components/WebhookMatchList.tsx
create mode 100644 airflow-core/src/airflow/ui/src/pages/Dag/Details.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/TaskInstance/CancelConditions.test.tsx
create mode 100644
airflow-core/src/airflow/ui/src/pages/TaskInstance/CancelConditions.tsx
copy airflow-core/src/airflow/ui/{rules/levels.js =>
src/utils/webhookMatch.ts} (68%)
create mode 100644 airflow-core/src/airflow/webhooks/cancel_application.py
create mode 100644 airflow-core/src/airflow/webhooks/cancel_requests.py
create mode 100644 airflow-core/tests/unit/models/test_dag_cancel_condition.py
create mode 100644
airflow-core/tests/unit/models/test_webhook_cancel_request.py
create mode 100644
airflow-core/tests/unit/serialization/definitions/test_webhook_match.py
create mode 100644 airflow-core/tests/unit/webhooks/test_cancel_application.py
create mode 100644 airflow-core/tests/unit/webhooks/test_cancel_requests.py
create mode 100644 dev/aip125_demo/bench_dags/cancel_bench.py
create mode 100644 dev/aip125_demo/dags/preview_environment.py
create mode 100644 dev/aip125_demo/dags/preview_with_teardown.py
create mode 100644 dev/aip125_demo/dags/ship_when_labeled.py
copy dev/aip125_demo/payloads/{github_pull_request_labeled.json =>
github_pull_request_synchronize.json} (82%)
copy airflow-core/src/airflow/webhooks/waits.py =>
shared/webhook_naming/src/airflow_shared/webhook_naming/matching.py (55%)
copy airflow-core/tests/unit/webhooks/test_waits.py =>
shared/webhook_naming/tests/webhook_naming/test_matching.py (73%)
create mode 100644 task-sdk/src/airflow/sdk/definitions/webhooks.py
create mode 100644 task-sdk/tests/task_sdk/definitions/test_webhooks.py
