This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fork-ci-test-merge-commit in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a85f022b85f9b3fcea5a4ad6a8f07c0637b1b315 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 16 01:48:11 2026 +0900 Test fork CI pull requests merged with their base branch Fork CI tested the pushed commit as it is, while project CI tests the pull request merged into its base branch, so a branch on an old base could go green on code that was never tested merged. The tested base check only bounded that gap. Each job now merges the pushed commit with the base branch commit build-info fixed for the run, so fork CI tests what project CI would, and the monitor drops the overlap rule for a run that reports and verifies its merge. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq --- .github/actions/merge-with-base/action.yml | 76 ++++++++++ .github/workflows/additional-ci-image-checks.yml | 11 ++ .github/workflows/additional-prod-image-tests.yml | 41 ++++++ .github/workflows/agent-framework-tests.yml | 11 ++ .github/workflows/airflow-distributions-tests.yml | 11 ++ .github/workflows/airflow-e2e-tests.yml | 11 ++ .github/workflows/basic-tests.yml | 52 ++++++- .github/workflows/ci-amd.yml | 80 +++++++++++ .github/workflows/ci-arm.yml | 80 +++++++++++ .github/workflows/ci-image-build.yml | 11 ++ .github/workflows/ci-image-checks.yml | 23 +++ .github/workflows/dependency-upgrade-summary.yml | 11 ++ .github/workflows/finalize-tests.yml | 12 ++ .github/workflows/generate-constraints.yml | 11 ++ .github/workflows/helm-tests.yml | 15 ++ .github/workflows/integration-system-tests.yml | 19 +++ .github/workflows/k8s-tests.yml | 15 ++ .github/workflows/kustomize-overlays-tests.yml | 11 ++ .github/workflows/openlineage-e2e-compat-tests.yml | 11 ++ .github/workflows/prod-image-build.yml | 12 ++ .github/workflows/prod-image-extra-checks.yml | 8 ++ .github/workflows/prod-packages-build.yml | 11 ++ .github/workflows/push-image-cache.yml | 15 ++ .github/workflows/run-unit-tests.yml | 11 ++ .github/workflows/special-tests.yml | 18 +++ .github/workflows/test-providers.yml | 15 ++ .github/workflows/ui-e2e-tests.yml | 11 ++ .pre-commit-config.yaml | 10 ++ contributing-docs/18_contribution_workflow.rst | 10 +- .../doc/ci/08_fork_ci_for_external_contributors.md | 128 +++++++++++------ .../src/airflow_breeze/utils/fork_ci_messages.py | 40 +++++- .../src/airflow_breeze/utils/fork_ci_reconcile.py | 134 ++++++++++++++++-- dev/breeze/tests/test_fork_ci_messages.py | 38 +++++ dev/breeze/tests/test_fork_ci_reconcile.py | 149 ++++++++++++++++++++ scripts/ci/prek/check_fork_ci_tests_merge.py | 156 +++++++++++++++++++++ .../ci/prek/test_check_fork_ci_tests_merge.py | 114 +++++++++++++++ 36 files changed, 1322 insertions(+), 70 deletions(-) diff --git a/.github/actions/merge-with-base/action.yml b/.github/actions/merge-with-base/action.yml new file mode 100644 index 00000000000..4d51f6fde9a --- /dev/null +++ b/.github/actions/merge-with-base/action.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Merge with the base branch' +description: >- + Replace the checked-out pushed commit of a fork CI run with its merge into the base branch, the + commit a pull_request run of project CI tests (AIP-120). Does nothing when merge-with is empty. +inputs: + merge-with: + description: >- + The base branch commit and the merge base with the pushed commit, space-separated, as + build-info resolved them for the whole run; empty outside fork runs. + required: true +outputs: + merge-sha: + description: "The merge commit now checked out; empty when nothing was merged." + value: ${{ steps.merge.outputs.merge-sha }} +runs: + using: "composite" + steps: + - name: "Merge the pushed commit with the base branch" + id: merge + if: inputs.merge-with != '' + shell: bash + env: + MERGE_WITH: ${{ inputs.merge-with }} + # The merge base is given, so the three commits are all that is fetched: no history between + # them is needed. Identity and dates are fixed, so every job of a run makes the same commit. + # The base branch is the first parent, as in GitHub's refs/pull/<n>/merge, so `<merge>^` + # is the base and `<merge>^..<merge>` is what the pull request changes. + run: | + set -euo pipefail + read -r base_sha merge_base <<< "${MERGE_WITH}" + head_sha="$(git rev-parse HEAD)" + for attempt in 1 2 3; do + if git fetch --no-tags --depth=1 https://github.com/apache/airflow.git \ + "${base_sha}" "${merge_base}"; then + break + fi + if [[ "${attempt}" -eq 3 ]]; then + echo "::error::Could not fetch ${base_sha} and ${merge_base} from apache/airflow" + exit 1 + fi + sleep $((attempt * 5)) + done + if ! tree="$(git merge-tree --write-tree --merge-base "${merge_base}" \ + "${base_sha}" "${head_sha}")"; then + echo "::error::The pushed commit conflicts with apache/airflow ${base_sha}; rebase it." + exit 1 + fi + commit_date="$(git show -s --format=%cI "${head_sha}")" + merge_sha="$( + GIT_AUTHOR_NAME="Fork CI" GIT_AUTHOR_EMAIL="[email protected]" \ + GIT_COMMITTER_NAME="Fork CI" GIT_COMMITTER_EMAIL="[email protected]" \ + GIT_AUTHOR_DATE="${commit_date}" GIT_COMMITTER_DATE="${commit_date}" \ + git commit-tree "${tree}" -p "${base_sha}" -p "${head_sha}" \ + -m "Merge ${head_sha} into apache/airflow ${base_sha}" + )" + git checkout --quiet --detach "${merge_sha}" + echo "merge-sha=${merge_sha}" >> "${GITHUB_OUTPUT}" + echo "Testing ${merge_sha}: ${head_sha} merged into apache/airflow ${base_sha}" diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index e1dbdd2a4e8..30778463455 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,6 +20,13 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -81,6 +88,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Check that image builds quickly" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index aca622255be..b0bd58a959d 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,6 +20,13 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -137,6 +144,7 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" @@ -154,6 +162,7 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" @@ -186,6 +195,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -225,6 +238,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -256,6 +273,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -274,6 +295,7 @@ jobs: if: inputs.run-generic-prod-image-tests == 'true' uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Regular e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -284,6 +306,7 @@ jobs: name: "Remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Remote logging e2e test" runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} @@ -298,6 +321,7 @@ jobs: name: "Elasticsearch remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Elasticsearch remote logging e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -312,6 +336,7 @@ jobs: name: "OpenSearch remote logging tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "OpenSearch remote logging e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -327,6 +352,7 @@ jobs: if: inputs.run-generic-prod-image-tests == 'true' uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "XCom object storage backend e2e test" runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} @@ -338,6 +364,7 @@ jobs: name: "Event driven tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Event driven e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -352,6 +379,7 @@ jobs: name: "Java SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Java SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -366,6 +394,7 @@ jobs: name: "Go SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Go SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -380,6 +409,7 @@ jobs: name: "TypeScript SDK e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "TypeScript SDK e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -394,6 +424,7 @@ jobs: name: "YAML Dag e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "YAML Dag e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -408,6 +439,7 @@ jobs: name: "Language SDK XCom object storage backend tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Language SDK XCom object storage backend e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -423,6 +455,7 @@ jobs: name: "OpenLineage e2e tests with PROD image" uses: ./.github/workflows/airflow-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "OpenLineage e2e test" runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} @@ -444,6 +477,7 @@ jobs: compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} uses: ./.github/workflows/openlineage-e2e-compat-tests.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" @@ -455,6 +489,7 @@ jobs: name: "Chromium UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Chromium UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -467,6 +502,7 @@ jobs: name: "Firefox UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "Firefox UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -479,6 +515,7 @@ jobs: name: "WebKit UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml with: + merge-with: ${{ inputs.merge-with }} workflow-name: "WebKit UI e2e tests" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} @@ -506,6 +543,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/agent-framework-tests.yml b/.github/workflows/agent-framework-tests.yml index 2f95c34cdfe..a9d065d7d0e 100644 --- a/.github/workflows/agent-framework-tests.yml +++ b/.github/workflows/agent-framework-tests.yml @@ -20,6 +20,13 @@ name: Agent framework tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -75,6 +82,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index d3ff1fb4af8..2ed3ec4eaf8 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -20,6 +20,13 @@ name: Non-core Distribution tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string # Static inputs defined to choose which distribution to test to run runners: description: "The array of labels (in json form) determining runners." @@ -95,6 +102,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/airflow-e2e-tests.yml b/.github/workflows/airflow-e2e-tests.yml index 77f3aa5b81c..3d513eb3c85 100644 --- a/.github/workflows/airflow-e2e-tests.yml +++ b/.github/workflows/airflow-e2e-tests.yml @@ -59,6 +59,13 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string workflow-name: description: "Name of the test" type: string @@ -112,6 +119,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index ac58cbf35b0..e23c8582a9f 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,6 +20,13 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string newsfragment-pr-number: description: "Pull request the newsfragments are checked against; empty to accept any number" required: false @@ -116,6 +123,10 @@ jobs: # Need to fetch all history for selective checks tests fetch-depth: 0 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Run unit tests" @@ -134,6 +145,10 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Install SVN" @@ -163,6 +178,10 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" @@ -204,6 +223,10 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" @@ -234,6 +257,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -270,6 +297,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Check translation completeness" @@ -306,6 +337,10 @@ jobs: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Fetch the last tested push commit" if: inputs.push-base-commit != '' shell: bash @@ -322,8 +357,11 @@ jobs: run: | set -euo pipefail prek_args=(--show-diff-on-failure --color always) + # A fork run checks out its merge with the base branch, whose first parent is the base, so + # the pull request's own changes are checked rather than only its last commit's. + tested="$(git rev-parse HEAD)" if [[ -z "${PUSH_BASE_COMMIT}" ]]; then - prek "${prek_args[@]}" --from-ref "${GITHUB_SHA}^" --to-ref "${GITHUB_SHA}" + prek "${prek_args[@]}" --from-ref "${tested}^" --to-ref "${tested}" elif git cat-file -e "${PUSH_BASE_COMMIT}^{commit}"; then mapfile -t files < <(git diff --name-only --no-renames --diff-filter=d \ "${PUSH_BASE_COMMIT}" "${GITHUB_SHA}") @@ -363,6 +401,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} test-airflow-release-commands: timeout-minutes: 80 @@ -383,6 +425,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Cleanup dist files" @@ -431,6 +477,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index f16449144bf..e4c12affa35 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -423,6 +423,9 @@ jobs: use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} + # The base branch commit and merge base that every job of a fork run merges the pushed + # commit with, so that it tests what a pull_request run of project CI tests; empty elsewhere. + merge-with: ${{ steps.merge-base.outputs.merge-with }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -435,6 +438,7 @@ jobs: fetch-depth: 2 persist-credentials: false - name: "Check clean integration with upstream target branch" + id: merge-base if: github.event_name == 'push' && github.repository != 'apache/airflow' shell: bash run: | @@ -467,6 +471,14 @@ jobs: exit 1 elif [ "${conflict}" -gt 1 ]; then echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + else + target_sha=$(git rev-parse "refs/remotes/upstream/${target_branch}") + { + echo "merge-with=${target_sha} ${merge_base}" + echo "target-branch=${target_branch}" + echo "target-sha=${target_sha}" + echo "merge-base=${merge_base}" + } >> "${GITHUB_OUTPUT}" fi else echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." @@ -474,6 +486,13 @@ jobs: else echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." fi + # The fork CI monitor reads from this step's name which merge the run tested. + - name: >- + Test merged with apache/airflow ${{ steps.merge-base.outputs.target-branch }}@${{ + steps.merge-base.outputs.target-sha }}, merge base ${{ steps.merge-base.outputs.merge-base }} + if: steps.merge-base.outputs.merge-with != '' + shell: bash + run: echo "The jobs of this run test the pushed commit merged with apache/airflow." - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. @@ -574,6 +593,7 @@ jobs: needs: [build-info, check-run-needed] uses: ./.github/workflows/basic-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} @@ -604,6 +624,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} push-image: "false" @@ -629,6 +650,7 @@ jobs: packages: write id-token: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -644,6 +666,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.generate-constraints == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -665,6 +688,7 @@ jobs: id-token: write contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} @@ -710,6 +734,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -752,6 +780,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -787,6 +819,7 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -812,6 +845,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} @@ -831,6 +865,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -864,6 +899,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -893,6 +929,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -922,6 +959,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -952,6 +990,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -981,6 +1020,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1012,6 +1052,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1043,6 +1084,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1073,6 +1115,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -1108,6 +1151,7 @@ jobs: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || needs.build-info.outputs.full-tests-needed == 'true') with: + merge-with: ${{ needs.build-info.outputs.merge-with }} default-branch: ${{ needs.build-info.outputs.default-branch }} runners: ${{ needs.build-info.outputs.runner-type }} in-memory-runners: ${{ needs.build-info.outputs.in-memory-runner-type }} @@ -1138,6 +1182,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} @@ -1162,6 +1207,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1192,6 +1238,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1220,6 +1267,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1237,6 +1285,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} build-type: "Regular" @@ -1260,6 +1309,7 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-branch: ${{ needs.build-info.outputs.default-branch }} @@ -1299,6 +1349,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -1322,6 +1373,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1354,6 +1406,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ @@ -1417,6 +1473,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -1459,6 +1519,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup Java @@ -1489,6 +1553,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # Serializes the same Dags with Airflow, from airflow-core's sources, to compare them. @@ -1545,6 +1613,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build Java SDK Javadoc" @@ -1577,6 +1649,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build TypeScript SDK API reference" @@ -1598,6 +1674,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '4+8', '16+32') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1619,6 +1696,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1635,6 +1713,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1685,6 +1764,7 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 3aadb07674e..bf2423afb41 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -299,6 +299,9 @@ jobs: use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} + # The base branch commit and merge base that every job of a fork run merges the pushed + # commit with, so that it tests what a pull_request run of project CI tests; empty elsewhere. + merge-with: ${{ steps.merge-base.outputs.merge-with }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -311,6 +314,7 @@ jobs: fetch-depth: 2 persist-credentials: false - name: "Check clean integration with upstream target branch" + id: merge-base if: github.event_name == 'push' && github.repository != 'apache/airflow' shell: bash run: | @@ -343,6 +347,14 @@ jobs: exit 1 elif [ "${conflict}" -gt 1 ]; then echo "::warning::Git merge-tree exited with error code ${conflict} (not a merge conflict). Skipping conflict check." + else + target_sha=$(git rev-parse "refs/remotes/upstream/${target_branch}") + { + echo "merge-with=${target_sha} ${merge_base}" + echo "target-branch=${target_branch}" + echo "target-sha=${target_sha}" + echo "merge-base=${merge_base}" + } >> "${GITHUB_OUTPUT}" fi else echo "::warning::Could not find merge-base with upstream/${target_branch} after deepening up to 1000 commits. Skipping conflict check." @@ -350,6 +362,13 @@ jobs: else echo "::warning::Could not fetch upstream/${target_branch} from https://github.com/apache/airflow.git. Skipping conflict check." fi + # The fork CI monitor reads from this step's name which merge the run tested. + - name: >- + Test merged with apache/airflow ${{ steps.merge-base.outputs.target-branch }}@${{ + steps.merge-base.outputs.target-sha }}, merge base ${{ steps.merge-base.outputs.merge-base }} + if: steps.merge-base.outputs.merge-with != '' + shell: bash + run: echo "The jobs of this run test the pushed commit merged with apache/airflow." - name: "Fetch the last tested push commit" # Selective checks of a push diff against this commit. A two-commit diff only needs the # commit itself, not the history in between; if it cannot be fetched, all tests run. @@ -450,6 +469,7 @@ jobs: needs: [build-info, check-run-needed] uses: ./.github/workflows/basic-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} @@ -480,6 +500,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} push-image: "false" @@ -505,6 +526,7 @@ jobs: packages: write id-token: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -520,6 +542,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.generate-constraints == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -541,6 +564,7 @@ jobs: id-token: write contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} @@ -586,6 +610,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -628,6 +656,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -663,6 +695,7 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -688,6 +721,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} @@ -707,6 +741,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -740,6 +775,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -769,6 +805,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" @@ -798,6 +835,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -828,6 +866,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" @@ -857,6 +896,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -888,6 +928,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -919,6 +960,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -949,6 +991,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" @@ -984,6 +1027,7 @@ jobs: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || needs.build-info.outputs.full-tests-needed == 'true') with: + merge-with: ${{ needs.build-info.outputs.merge-with }} default-branch: ${{ needs.build-info.outputs.default-branch }} runners: ${{ needs.build-info.outputs.runner-type }} in-memory-runners: ${{ needs.build-info.outputs.in-memory-runner-type }} @@ -1014,6 +1058,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} @@ -1038,6 +1083,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1068,6 +1114,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" @@ -1096,6 +1143,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1113,6 +1161,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} build-type: "Regular" @@ -1136,6 +1185,7 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} default-branch: ${{ needs.build-info.outputs.default-branch }} @@ -1175,6 +1225,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -1198,6 +1249,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1230,6 +1282,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ @@ -1293,6 +1349,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm @@ -1335,6 +1395,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: Setup Java @@ -1365,6 +1429,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary # Serializes the same Dags with Airflow, from airflow-core's sources, to compare them. @@ -1421,6 +1489,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build Java SDK Javadoc" @@ -1453,6 +1525,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ needs.build-info.outputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Build TypeScript SDK API reference" @@ -1474,6 +1550,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '4+8', '16+32') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1495,6 +1572,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" @@ -1511,6 +1589,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1561,6 +1640,7 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: + merge-with: ${{ needs.build-info.outputs.merge-with }} runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 78d676715c4..e4d70b16aa1 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,6 +20,13 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -154,6 +161,10 @@ jobs: with: ref: ${{ inputs.checkout-ref }} persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.checkout-ref == '' && inputs.merge-with || '' }} # A local action is read from the workspace, and `checkout-ref` may be a release ref cut # before the stash actions existed, so they are loaded from the workflow's own version. The # runner-preparing scripts too: an older `make_mnt_writeable.sh` empties /mnt, which on diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index dcadcd6dbfd..dc5c658629e 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -20,6 +20,13 @@ name: CI Image Checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string newsfragment-pr-number: description: "Pull request the newsfragments are checked against; empty to accept any number" required: false @@ -142,6 +149,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -214,6 +225,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -375,6 +390,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Download docs prepared as artifacts" @@ -510,6 +529,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/dependency-upgrade-summary.yml b/.github/workflows/dependency-upgrade-summary.yml index 626f594940c..ae827b1f32f 100644 --- a/.github/workflows/dependency-upgrade-summary.yml +++ b/.github/workflows/dependency-upgrade-summary.yml @@ -23,6 +23,13 @@ name: Dependency upgrade summary on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -64,6 +71,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 52926b1028d..d7571f3d152 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,6 +20,13 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -103,6 +110,10 @@ jobs: with: # Needed to perform push action persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "RunsOn cost summary" uses: ./.github/actions/runs-on-cost-summary - name: "Set constraints branch name" @@ -135,6 +146,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} cache-type: "Regular AMD" diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index b388af7196f..c7d5f5f9132 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,6 +20,13 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -116,6 +123,10 @@ jobs: with: ref: ${{ inputs.checkout-ref }} persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.checkout-ref == '' && inputs.merge-with || '' }} - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1f505b8a453..4c7a3c3be7a 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,6 +20,13 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -81,6 +88,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -112,6 +123,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Install Breeze" uses: ./.github/actions/breeze - name: Setup git for tagging diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index e46173b9c70..eb715b94dab 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -20,6 +20,13 @@ name: Integration and system tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public runners." required: true @@ -103,6 +110,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -156,6 +167,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -201,6 +216,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 90ce5a9ffee..2c24af024a0 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -20,6 +20,13 @@ name: K8s tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -94,6 +101,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous # step id: prepare-versions - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" @@ -174,6 +185,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous # step id: prepare-versions - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" diff --git a/.github/workflows/kustomize-overlays-tests.yml b/.github/workflows/kustomize-overlays-tests.yml index 5f9be6f52fb..245cc12015b 100644 --- a/.github/workflows/kustomize-overlays-tests.yml +++ b/.github/workflows/kustomize-overlays-tests.yml @@ -20,6 +20,13 @@ name: Kustomize overlays smoke tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -73,6 +80,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/openlineage-e2e-compat-tests.yml b/.github/workflows/openlineage-e2e-compat-tests.yml index 4b9cb6d4408..10906b68134 100644 --- a/.github/workflows/openlineage-e2e-compat-tests.yml +++ b/.github/workflows/openlineage-e2e-compat-tests.yml @@ -24,6 +24,13 @@ permissions: on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -73,6 +80,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" if: ${{ !startsWith(inputs.airflow-version, '2.') }} uses: ./.github/actions/prepare_breeze_and_image diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index a7c898ff7f1..c2f08049e56 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,6 +20,13 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -136,6 +143,7 @@ jobs: permissions: contents: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} default-python-version: ${{ inputs.default-python-version }} @@ -182,6 +190,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 5b3d072a49a..3afa932970f 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,6 +20,13 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -62,6 +69,7 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml with: + merge-with: ${{ inputs.merge-with }} runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} build-type: "pip" diff --git a/.github/workflows/prod-packages-build.yml b/.github/workflows/prod-packages-build.yml index c80bfd816db..2b57e1d75a3 100644 --- a/.github/workflows/prod-packages-build.yml +++ b/.github/workflows/prod-packages-build.yml @@ -23,6 +23,13 @@ name: Build PROD packages on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -66,6 +73,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Make /mnt writeable" run: ./scripts/ci/make_mnt_writeable.sh if: inputs.upload-package-artifact == 'true' diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 8fdb06f95ce..666ff3c8081 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,6 +20,13 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -117,6 +124,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" run: ./scripts/tools/free_up_disk_space.sh - name: "Install Breeze" @@ -187,6 +198,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Free up disk space" run: ./scripts/tools/free_up_disk_space.sh - name: "Install Breeze" diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index c2564f71462..53fbcc5f62f 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,6 +20,13 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public AMD runners." required: true @@ -199,6 +206,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Make /mnt writeable" run: ./scripts/ci/make_mnt_writeable.sh - name: "Move docker to /mnt" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 1b47ac5259a..7dfbf784c9d 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,6 +20,13 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining runners." required: true @@ -102,6 +109,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" @@ -127,6 +135,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" @@ -152,6 +161,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" @@ -177,6 +187,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" @@ -202,6 +213,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} upgrade-boto: "true" @@ -228,6 +240,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-boto: "true" @@ -254,6 +267,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" @@ -280,6 +294,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" @@ -306,6 +321,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} test-name: "Postgres" @@ -331,6 +347,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '4+8', '16+32') }} platform: ${{ inputs.platform }} test-name: "Postgres" @@ -356,6 +373,7 @@ jobs: contents: read packages: read with: + merge-with: ${{ inputs.merge-with }} runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} test-name: "SystemTest" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index 19780189cb3..b0171693834 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -20,6 +20,13 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string runners: description: "The array of labels (in json form) determining public AMD runners." required: true @@ -96,6 +103,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: @@ -207,6 +218,10 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index 96e68f83e12..4b1417b0b1e 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -55,6 +55,13 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: + merge-with: + description: >- + The base branch commit and merge base to merge the pushed commit with, space-separated; + empty outside fork CI runs. + required: false + default: "" + type: string workflow-name: description: "Name of the test" type: string @@ -107,6 +114,10 @@ jobs: with: fetch-depth: 2 persist-credentials: false + - name: "Merge with the base branch" + uses: ./.github/actions/merge-with-base + with: + merge-with: ${{ inputs.merge-with }} - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5aa9c046bd0..e1bc54aa561 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1740,6 +1740,16 @@ repos: pass_filenames: true require_serial: true files: ^\.github/workflows/.*\.yml$ + - id: check-fork-ci-tests-merge + name: Check fork CI jobs test the pushed commit merged with the base branch + env: + PYTHONPYCACHEPREFIX: .build/pycache + description: Every job of a fork run that checks out the pushed commit merges it with the base branch + entry: ./scripts/ci/prek/check_fork_ci_tests_merge.py + language: python + pass_filenames: false + require_serial: true + files: ^\.github/workflows/.*\.yml$|^scripts/ci/prek/check_fork_ci_tests_merge\.py$ - id: check-docstring-param-types name: Check that docstrings do not specify param types env: diff --git a/contributing-docs/18_contribution_workflow.rst b/contributing-docs/18_contribution_workflow.rst index 8c2e23990f6..14c0c8b4b50 100644 --- a/contributing-docs/18_contribution_workflow.rst +++ b/contributing-docs/18_contribution_workflow.rst @@ -197,11 +197,11 @@ does not run for 30 days is closed with an explanation; you can reopen it after files that decide which CI jobs run and where, such as ``.github/``, the breeze sources or ``scripts/ci/prek/``, a green fork run is not enough: a maintainer reviews the change and runs project CI for it. -Fork CI tests your branch as it is, while project CI tests it merged with the target branch. The -``CI passed`` check that merging requires is therefore reported only when your branch is fewer than -200 commits behind the target branch and the target branch has not changed any file your pull request -changes since your branch's base. Otherwise the pull request comment lists those files and asks you -to rebase on ``upstream/main`` (or the target branch) and push, which runs fork CI again. +Like project CI, fork CI tests your branch merged with the target branch, as the target branch was +when the run started; a push that conflicts with it fails at once and asks you to rebase. The +``CI passed`` check that merging requires is reported only when that commit of the target branch is +fewer than 200 commits behind the target branch. Otherwise the pull request comment asks you to +rebase on ``upstream/main`` (or the target branch) and push, which runs fork CI again. **Joining the pilot.** Open a pull request that adds your GitHub login to the ``[pilot]`` table in ``.github/fork-ci-gate.toml``, with a join instant after the expected merge, for example the next day diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index a1d932971a5..b9faf3646c8 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -464,11 +464,13 @@ upstream yields no changed files, so such a run performs basic checks only. would be the contributor's own previous tip after they merge upstream `main`, so a release manager's bump from `main` would count as theirs. A file missing at the merge base is new; any other read failure fails the run rather than letting the check pass unseen. -- **Early conflict check**: before the test matrix runs, `ci-amd.yml` verifies that the pushed - commit merges cleanly with upstream's target branch using `git merge-tree`. It fetches the - upstream target branch and deepens history in increments of 200 commits up to 1,000 commits if +- **Early conflict check and merge base**: before the test matrix runs, `build-info` verifies that + the pushed commit merges cleanly with upstream's target branch using `git merge-tree`. It fetches + the upstream target branch and deepens history in increments of 200 commits up to 1,000 commits if needed to resolve the merge base; if a merge conflict is detected, the workflow fails early with - clear rebase instructions rather than spending runner hours on conflicting code. + clear rebase instructions rather than spending runner hours on conflicting code. Otherwise it + outputs the target branch commit and the merge base as `merge-with`, which every later job uses to + test the pushed commit merged with that commit (see "Testing the merged result"). - **Labels** come from the open `apache/airflow` pull request whose head is `<fork-owner>:<branch>`, since a push event carries none. `full tests needed`, `allow provider dependency bump`, `skip common compat check` and the other labels work as on @@ -480,6 +482,38 @@ upstream yields no changed files, so such a run performs basic checks only. lands. - Canary detection is unchanged and already false outside `apache/airflow`. +### Testing the merged result + +A `pull_request` run of project CI tests GitHub's merge commit of the pull request into its base +branch. A fork run is a `push` run of the pushed commit, so every job that checks out the pull +request's sources turns that checkout into the same kind of merge before it builds or tests +anything, with the `merge-with-base` action (`.github/actions/merge-with-base`): + +- `build-info` fixes the merge for the whole run: the target branch commit it fetched for the + conflict check and the merge base with the pushed commit, output as `merge-with`. Every job + therefore merges with the same commit, even when the target branch moves during the run. +- Each job fetches those two commits from `apache/airflow` at depth 1 and builds the merge with + `git merge-tree --write-tree --merge-base`, which needs the three commits and none of the history + between them, so the shallow checkouts the jobs already use are enough. `git commit-tree` turns the + tree into a commit with the target branch commit as its first parent and the pushed commit as its + second, as in GitHub's `refs/pull/<n>/merge`, so `<merge>^..<merge>` is what the pull request + changes; the basic static checks diff exactly that. The author, committer and dates are fixed, + so every job makes the same merge commit, which is then checked out in place of the pushed commit. +- The merge needs no write access: nothing is pushed, and fork runs keep a read-only token. +- `merge-with` is passed to every reusable workflow a fork run calls, and is empty in every run in + `apache/airflow`, where the action does nothing. A workflow whose checkout takes another ref + (`checkout-ref`, such as a target branch to build an image from) merges only when it checks out + the pushed commit. +- The `check-fork-ci-tests-merge` prek hook keeps it that way: in `ci-amd.yml`, `ci-arm.yml` and every + workflow they call, each call passes `merge-with`, and each job that checks out the pushed commit + into the workspace merges after its last such checkout. `gate`, `build-info`, `notify-slack` and + `summarize-warnings` are exempt: they read the pushed commit or only report on the run. +- `build-info` names the merge in a step, "Test merged with apache/airflow + `<branch>`@`<commit>`, merge base `<commit>`", from which the reconciler learns what the run + tested (see "Tested base"). When the conflict check cannot fetch the target branch or find the + merge base, `merge-with` is empty, the jobs test the pushed commit as it is, and the step is + skipped. + `breeze ci selective-check` gets the compare result through a new `--changed-files-from-api` path taken automatically when `is_fork_push` holds; the `git diff-tree` path stays for every other event. The `GITHUB_TOKEN` of the fork run is sufficient: the compare reads public data. @@ -1151,14 +1185,27 @@ also run in a gated run started, none failed, and at least one was cancelled. ### Tested base -A fork run tests the pushed commit as it is, while project CI's `pull_request` run tests GitHub's -merge commit of the pull request into its base branch. A fork run on an old base could otherwise go -green and merge although the merged result was never tested. So `CI passed` is only reported for a -fork run whose base is close enough to the base branch that the merge changes nothing the pull -request touches. +A fork run tests the pushed commit merged with a commit of the base branch ("Testing the merged +result"), as project CI's `pull_request` run tests GitHub's merge commit. The base branch moves on +after the run, so `CI passed` is only reported for a run whose merge is recent enough. + +When a head commit turns green, the reconciler reads the merge the run tested from the step of its +`Build info` job that names it, and checks it against GitHub, once per head commit: + +- the named branch must be the pull request's base branch; +- `GET /repos/apache/airflow/compare/<tested commit>...<fork-owner>:<sha>` must give the named merge + base, so the merge is the one GitHub would compute for the head commit; +- `GET /repos/apache/airflow/compare/<tested commit>...<base>` must say the base branch is + `ahead` of the tested commit or `identical` to it, so the tested commit is on the base branch, and + its `ahead_by` is how far behind the merge is (`behind`). + +The result is stored in the marker as `"base": {"sha", "behind", "overlap": [], "merged": true}`. +The run's report is trusted as far as its green result is: both come from the workflow of the head +commit, which a pull request changing CI-defining files is not trusted with in any case (see +"Trusting a green fork run"). -When a head commit turns green, the reconciler compares it with the base branch, once per head -commit, and stores the result in the marker as `"base": {"sha", "behind", "overlap"}`: +A run that names no merge (the conflict check could not resolve one), or whose merge does not +check out, is compared as a run of the head commit as it is, as before merging was added: - `GET /repos/apache/airflow/compare/<base>...<fork-owner>:<sha>` gives the merge base, the commit of the base branch the fork run tested on top of (`sha`), the commits the base branch has since @@ -1168,27 +1215,26 @@ commit, and stores the result in the marker as `"base": {"sha", "behind", "overl counting under both paths. `CI passed` is reported only when the base is current: fewer than `MAX_BEHIND_COMMITS` (200, about -four days of `main`) commits behind and no overlap. Otherwise the state stays `green` (the pull -request can be reviewed, and its status, labels and draft state are those of `green`), the state -comment lists the overlapping files or says how far behind the base is and gives the rebase -commands, and `CI passed` is posted as `pending` with a description saying to rebase. A rebase and -push starts a new fork run, which is compared again when it turns green. Every green state comment -names the base commit the run tested and how far behind it was, and the `CI passed` description -names it as well. - -The compare API lists at most 300 files. A pull request with more is listed with -`GET /repos/apache/airflow/pulls/<n>/files` instead (up to 3000); a base branch that changed 300 or -more files since the merge base cannot be checked and counts as overlapping, which a rebase -resolves. When the compare fails, `CI passed` is not reported, the failure is reported to Slack and -the next run compares again. A marker written before this check, with no `base`, is compared again -on the next run, so a pull request that already had `CI passed` loses it if its base turns out not -to be current. A retarget compares again, as it decides green trust again. - -This narrows the gap to project CI without closing it. A base change in other files can still break -the pull request, within at most 200 commits, which matches what project CI leaves open after its -run: a `pull_request` run is not repeated when the base branch moves, so the merged result at merge -time is not the one it tested either. Testing the merge commit in the fork itself was considered and -rejected; see "Known limitations". +four days of `main`) commits behind, and, for a run that did not test the merge, no overlap. +Otherwise the state stays `green` (the pull request can be reviewed, and its status, labels and +draft state are those of `green`), the state comment says how far behind the base is or lists the +overlapping files and gives the rebase commands, and `CI passed` is posted as `pending` with a +description saying to rebase. A rebase and push starts a new fork run, which is compared again when +it turns green. Every green state comment names the base commit the run tested, whether merged +with it or on top of it, and how far behind it was, and the `CI passed` description names it as +well. + +The compare API lists at most 300 files. For a run that did not test the merge, a pull request with +more is listed with `GET /repos/apache/airflow/pulls/<n>/files` instead (up to 3000); a base branch +that changed 300 or more files since the merge base cannot be checked and counts as overlapping, +which a rebase resolves. When the compare fails, `CI passed` is not reported, the failure is +reported to Slack and the next run compares again. A marker written before this check, with no +`base`, is compared again on the next run, so a pull request that already had `CI passed` loses it +if its base turns out not to be current. A retarget compares again, as it decides green trust again. + +A merged run leaves the same gap as project CI: a `pull_request` run is not repeated when the base +branch moves, so the merged result at merge time is not the one it tested either. The 200-commit +limit bounds that gap, which project CI does not. ### Unknown failures and attention items go to Slack @@ -1443,17 +1489,13 @@ as a failed check rather than a crash. - An external pull request that legitimately changes workflow files, was released with `use project ci` and then had that label removed is closed on the next run, because both signals hold. Keeping `use project ci` on such a pull request is the intended handling. -- Fork CI tests the head commit, not the pull request merged into its base branch; "Tested base" - only bounds the difference. Testing the merged result in the fork was rejected because: - every job of the fork run checks out the pushed commit (about 60 `actions/checkout` steps across - `ci-amd.yml` and its reusable workflows, plus the image builds and caches keyed by the commit), - and each would have to check out one merge commit fixed for the whole run; the first push of a - branch happens before its pull request exists, so there is no `refs/pull/<n>/merge` to test yet; - GitHub computes that ref asynchronously after each push and replaces it whenever the base branch - moves, so a run would have to wait for it and could find its commit gone mid-run; and the - reconciler would have to trust the run's own report of which merge it tested. Merging the base - branch into the head in every job instead needs the shared history of both, which shallow - checkouts do not have. +- A fork run merges with the target branch that breeze's default branch at the pushed commit names + (`main`, or the release branch it was cut from), since a push does not know which pull request it + is for. A pull request to another base branch is compared as a run of the head commit as it is + ("Tested base"). A run whose conflict check cannot fetch the target branch, or find the merge base + within 1,000 commits, also tests the head commit as it is. +- Every job of a fork run fetches the target branch commit and the merge base (two depth-1 fetches, + some seconds each) to build the merge, about 55 jobs for a full-matrix run. ## Testing diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py index 7378ad7f368..daaae9783ef 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py @@ -55,9 +55,9 @@ CI_PASSED_PROJECT_DESCRIPTION = "Project CI passed for this commit" CI_RUN_STARTED_CONTEXT = "CI run started" CI_RUN_STARTED_DESCRIPTION = "CI runs in the contributor's fork - details open the run" CI_RUN_STARTED_PROJECT_DESCRIPTION = "Project CI runs in apache/airflow - details open the run" -# A fork run tests the head commit as it is, not merged with the base branch as project CI does. From -# this many commits behind, `CI passed` is withheld until the branch is rebased, as it is when the -# base branch changed any file the pull request changes. +# From this many commits behind the base branch commit a fork run tested with, `CI passed` is withheld +# until the branch is rebased. A run that tested the head commit as it is, not merged with the base +# branch, also gets none when the base branch has since changed a file the pull request changes. MAX_BEHIND_COMMITS = 200 # Overlapping files named in the state comment; the marker keeps them all. OVERLAP_LISTED = 10 @@ -97,20 +97,24 @@ class BaseCheck: """ The base a green fork run tested, compared with the base branch when the run turned green. - :param sha: the merge base of the head commit and the base branch, the base the run tested. + :param sha: the base branch commit the run tested the head commit merged with; for a run that + tested the head commit as it is, the merge base of the head commit and the base branch. :param behind: commits on the base branch since ``sha``. :param overlap: paths that both the pull request and the base branch since ``sha`` change; - None when GitHub listed too many changed files on either side to tell. + None when GitHub listed too many changed files on either side to tell. Not checked for a + merged run, which tested the base branch's changes until ``sha`` with the pull request. + :param merged: whether the run tested the head commit merged with ``sha``, as project CI does. """ sha: str behind: int overlap: tuple[str, ...] | None = () + merged: bool = False @property def is_current(self) -> bool: """Whether the run tested close enough to the merged result for `CI passed`.""" - return self.behind < MAX_BEHIND_COMMITS and self.overlap == () + return self.behind < MAX_BEHIND_COMMITS and (self.merged or self.overlap == ()) @dataclass(frozen=True) @@ -132,11 +136,14 @@ class Marker: def render_marker(marker: Marker) -> str: data: dict[str, object] = {"state": marker.state, "sha": marker.sha, "since": marker.since.isoformat()} if marker.base is not None: - data["base"] = { + base: dict[str, object] = { "sha": marker.base.sha, "behind": marker.base.behind, "overlap": None if marker.base.overlap is None else list(marker.base.overlap), } + if marker.base.merged: + base["merged"] = True + data["base"] = base if marker.jobs is not None: data["jobs"] = marker.jobs if marker.failed_run is not None: @@ -165,6 +172,7 @@ def parse_marker(body: str) -> Marker | None: sha=base["sha"], behind=base["behind"], overlap=None if base["overlap"] is None else tuple(base["overlap"]), + merged=base.get("merged") is True, ), jobs=data.get("jobs"), failed_run=data.get("failed_run"), @@ -290,6 +298,20 @@ def strip_failed_job_list(body: str) -> str: def tested_base_text(base: BaseCheck, *, base_ref: str) -> str: + if base.merged: + text = ( + f"Fork CI tested this branch merged with `{base_ref}` at " + f"[`{base.sha[:10]}`](https://github.com/apache/airflow/commit/{base.sha}), as project CI " + f"would, {base.behind} commits behind `{base_ref}` when the run turned green." + ) + if base.is_current: + return text + return ( + f"{text}\n\n**Rebase needed.** That is too far behind to say much about the merged result. " + f"The `CI passed` check is not reported until fork CI is green for a branch rebased on " + f"`{base_ref}`:\n\n" + f"```bash\ngit fetch upstream\ngit rebase upstream/{base_ref}\ngit push --force-with-lease\n```" + ) text = ( f"Fork CI tested this branch on top of `{base_ref}` at " f"[`{base.sha[:10]}`](https://github.com/apache/airflow/commit/{base.sha}), " @@ -320,6 +342,10 @@ _REBASE_NEEDED_PREFIX = "Rebase on " def ci_passed_description(base: BaseCheck, *, base_ref: str) -> str: + if base.merged: + return ( + f"{_FORK_CI_PASSED_PREFIX}; merged with {base_ref}@{base.sha[:10]}, {base.behind} commits behind" + ) return ( f"{_FORK_CI_PASSED_PREFIX}; base {base_ref}@{base.sha[:10]} was {base.behind} commits behind, " "no overlap" diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index 61a30366219..37e7d10d25b 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -36,7 +36,7 @@ from dataclasses import asdict, dataclass, field, replace from datetime import UTC, datetime, timedelta from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from urllib.parse import quote import yaml @@ -128,6 +128,13 @@ GATED_RUN_JOBS = ( ) GATED_RUN_JOB_NAMES = frozenset[str]().union(*GATED_RUN_JOBS) CHECK_RUN_NEEDED_JOB_NAME = "Check if the run is needed" +BUILD_INFO_JOB_NAME = "Build info" +# The step of build-info that names the base branch commit and merge base every job of a fork run +# merges the head commit with (ci-amd.yml). +TESTED_MERGE_STEP = re.compile( + r"^Test merged with apache/airflow (?P<base_ref>\S+)@(?P<base_sha>[0-9a-f]{40}), " + r"merge base (?P<merge_base>[0-9a-f]{40})$" +) PROJECT_CI_NOT_RUN_JOB_NAME = "CI passed (project CI not run)" CI_PASSED_CANCELLED_JOB_NAME = "CI passed (run cancelled)" BOT_LOGINS = frozenset( @@ -1895,31 +1902,72 @@ def find_failed_jobs(jobs: Iterable[dict[str, Any]]) -> list[FailedJob]: ReadJobs = dict[int, list[dict[str, Any]] | None] -def read_failed_jobs( +def read_run_jobs( client: GitHubClient, pr: PullRequestSnapshot, fork: ForkObservation, read_jobs: ReadJobs, -) -> list[FailedJob] | None: +) -> list[dict[str, Any]] | None: """ - ``find_failed_jobs`` of the pull request's fork run; None when its jobs cannot be read. + The jobs of the pull request's fork run; None when they cannot be read. :param read_jobs: the jobs already read in this reconciler run; a read made here is recorded too. """ if fork.run_id is None or not pr.head_repo: - return [] - if fork.run_id in read_jobs: - jobs = read_jobs[fork.run_id] - else: + return None + if fork.run_id not in read_jobs: try: - jobs = client.get_run_jobs(pr.head_repo, fork.run_id) + read_jobs[fork.run_id] = client.get_run_jobs(pr.head_repo, fork.run_id) except GitHubError as e: console_print(f"[warning]#{pr.number}: could not read the jobs of {fork.run_url}: {e}[/]") - jobs = None - read_jobs[fork.run_id] = jobs + read_jobs[fork.run_id] = None + return read_jobs[fork.run_id] + + +def read_failed_jobs( + client: GitHubClient, + pr: PullRequestSnapshot, + fork: ForkObservation, + read_jobs: ReadJobs, +) -> list[FailedJob] | None: + """``find_failed_jobs`` of the pull request's fork run; None when its jobs cannot be read.""" + if fork.run_id is None or not pr.head_repo: + return [] + jobs = read_run_jobs(client, pr, fork, read_jobs) return None if jobs is None else find_failed_jobs(jobs) +class ForkRunMerge(NamedTuple): + """The base branch commit a fork run merged the head commit with, as its build-info job names it.""" + + base_ref: str + base_sha: str + merge_base: str + + +def find_tested_merge(jobs: Iterable[dict[str, Any]]) -> ForkRunMerge | None: + """The merge a fork run tested, from the name of a step of its `Build info` job; None if it named none.""" + for job in jobs: + if job.get("name") != BUILD_INFO_JOB_NAME: + continue + for step in job.get("steps") or []: + match = TESTED_MERGE_STEP.match(step.get("name") or "") + if match and step.get("conclusion") == "success": + return ForkRunMerge(match["base_ref"], match["base_sha"], match["merge_base"]) + return None + + +def read_tested_merge( + client: GitHubClient, + pr: PullRequestSnapshot, + fork: ForkObservation, + read_jobs: ReadJobs, +) -> ForkRunMerge | None: + """``find_tested_merge`` of the pull request's fork run; None when its jobs cannot be read.""" + jobs = read_run_jobs(client, pr, fork, read_jobs) + return None if jobs is None else find_tested_merge(jobs) + + class JobStatusReporter: """Reports each job of a pull request's fork run as a commit status, within a per-run budget.""" @@ -2559,17 +2607,69 @@ def pull_request_changes_ci_files(client: GitHubClient, repo: str, number: int, return changes_ci_definition(changed_paths(files), file_count=len(files)) -def compare_with_base(client: GitHubClient, repo: str, pr: PullRequestSnapshot) -> BaseCheck | None: +def compare_tested_merge( + client: GitHubClient, repo: str, pr: PullRequestSnapshot, tested: ForkRunMerge +) -> BaseCheck | None: + """ + Check the merge a fork run reports it tested against GitHub; None when it does not hold. + + The run's own report is trusted as far as its green result is: both come from the workflow of + the head commit, which a pull request changing CI-defining files does not get trusted with. The + reported commit must still be on the pull request's base branch and have the reported merge base + with the head commit, so that the merge is the one project CI would test. + """ + if tested.base_ref != pr.base_ref: + console_print( + f"[warning]#{pr.number}: the fork run merged with {tested.base_ref}, not {pr.base_ref}[/]" + ) + return None + owner = (pr.head_repo or "").split("/", 1)[0] + base_ref = quote(pr.base_ref, safe="") + try: + head_side = client.rest_get( + f"repos/{repo}/compare/{tested.base_sha}...{owner}:{pr.head_sha}?per_page=1" + ) + base_side = client.rest_get(f"repos/{repo}/compare/{tested.base_sha}...{base_ref}?per_page=1") + merge_base = head_side["merge_base_commit"]["sha"] + status = base_side["status"] + behind = base_side["ahead_by"] + except (GitHubError, KeyError, TypeError) as e: + console_print(f"[warning]#{pr.number}: could not check the merge the fork run tested: {e}[/]") + return None + if merge_base != tested.merge_base or status not in ("ahead", "identical"): + console_print( + f"[warning]#{pr.number}: the fork run's merge with {tested.base_sha} does not match GitHub " + f"(merge base {merge_base}, {pr.base_ref} {status}); comparing its head commit instead[/]" + ) + return None + return BaseCheck(tested.base_sha, behind, (), merged=True) + + +def compare_with_base( + client: GitHubClient, + repo: str, + pr: PullRequestSnapshot, + *, + find_tested_merge: Callable[[], ForkRunMerge | None] = lambda: None, +) -> BaseCheck | None: """ Compare the base a fork run tested with the base branch; None when not readable. - The fork run tests the head commit, so its base is the merge base with the base branch. Two + A fork run that reports the merge it tested is checked with ``compare_tested_merge``. Otherwise + it tested the head commit, so its base is the merge base with the base branch. Two three-dot compares from it give the files the pull request changes and the files the base branch changed since; a file in both means the merged result differs from what the fork run tested in a file the pull request changes. Compares list at most 300 files (on their first page, which ``per_page`` does not limit): a pull request with more is listed with the files endpoint instead, while a base branch that changed more cannot be checked until the branch is rebased. + + :param find_tested_merge: the merge the fork run reports it tested, None when it reports none. """ + tested = find_tested_merge() + if tested is not None: + merged = compare_tested_merge(client, repo, pr, tested) + if merged is not None: + return merged owner = (pr.head_repo or "").split("/", 1)[0] base_ref = quote(pr.base_ref, safe="") try: @@ -2956,7 +3056,13 @@ def run_reconcile( changes_ci_files=functools.partial( pull_request_changes_ci_files, client, repo, pr.number, pr.head_sha ), - compare_base=functools.partial(compare_with_base, client, repo, pr), + compare_base=functools.partial( + compare_with_base, + client, + repo, + pr, + find_tested_merge=functools.partial(read_tested_merge, client, pr, fork, read_jobs), + ), jobs_reported=jobs_reported, head_shared_across_bases=pr.head_sha in shared_heads, find_gate_change=functools.partial(gate_change_finder.find, pr), diff --git a/dev/breeze/tests/test_fork_ci_messages.py b/dev/breeze/tests/test_fork_ci_messages.py index d945dfe3576..283199209ff 100644 --- a/dev/breeze/tests/test_fork_ci_messages.py +++ b/dev/breeze/tests/test_fork_ci_messages.py @@ -69,6 +69,7 @@ MARKER = Marker(state="awaiting_push", sha="abc123", since=datetime(2026, 10, 15 pytest.param(MARKER, id="without-base"), pytest.param(replace(MARKER, base=BaseCheck("def456", 12, ("a.py",))), id="base"), pytest.param(replace(MARKER, base=BaseCheck("def456", 12, None)), id="base-unchecked"), + pytest.param(replace(MARKER, base=BaseCheck("def456", 12, merged=True)), id="base-merged"), pytest.param(replace(MARKER, jobs="123:SUCCESS"), id="jobs"), pytest.param(replace(MARKER, failed_run="123:2"), id="failed-run"), pytest.param(replace(MARKER, gate_checked=True), id="gate-checked"), @@ -83,6 +84,10 @@ def test_marker_without_base_keeps_its_rendering(): assert "jobs" not in render_marker(MARKER) +def test_marker_of_an_unmerged_base_keeps_its_rendering(): + assert "merged" not in render_marker(replace(MARKER, base=BaseCheck("def456", 12))) + + def test_marker_from_before_the_base_check_has_no_base(): marker = parse_marker( '<!-- fork-ci-gate: {"behind": 3, "sha": "a", "since": "2026-10-15T10:00:00+00:00", "state": "green"} -->' @@ -99,6 +104,9 @@ def test_marker_from_before_the_base_check_has_no_base(): pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS), False, id="far-behind"), pytest.param(BaseCheck("d", 1, ("a.py",)), False, id="overlap"), pytest.param(BaseCheck("d", 1, None), False, id="unchecked"), + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS - 1, merged=True), True, id="merged-close"), + pytest.param(BaseCheck("d", MAX_BEHIND_COMMITS, merged=True), False, id="merged-far-behind"), + pytest.param(BaseCheck("d", 1, None, merged=True), True, id="merged-overlap-not-checked"), ], ) def test_base_check_is_current(base, current): @@ -142,6 +150,24 @@ def test_base_check_is_current(base, current): pytest.param( "green", BaseCheck("d", 4, None), ["Rebase needed", "too many files"], [], id="unchecked" ), + pytest.param( + "green", + BaseCheck("d" * 40, 3, merged=True), + [ + "merged with `v3-1-test` at [`dddddddddd`](https://github.com/apache/airflow/commit/" + + "d" * 40, + "as project CI would, 3 commits", + ], + ["Rebase needed", "as it is"], + id="merged", + ), + pytest.param( + "green", + BaseCheck("d", MAX_BEHIND_COMMITS, merged=True), + ["merged with", "Rebase needed", "too far behind", "git rebase upstream/v3-1-test"], + [], + id="merged-far-behind", + ), pytest.param("failed", BaseCheck("d", MAX_BEHIND_COMMITS), [], ["tested this branch"], id="failed"), pytest.param("green", None, [], ["tested this branch"], id="not-compared"), ], @@ -230,6 +256,11 @@ def test_strip_failed_job_list_leaves_the_comment_without_the_list(): [ pytest.param("Fork CI passed for this commit", True, id="before-the-base-check"), pytest.param(ci_passed_description(BaseCheck("a" * 40, 3), base_ref="main"), True, id="current-base"), + pytest.param( + ci_passed_description(BaseCheck("a" * 40, 3, merged=True), base_ref="main"), + True, + id="merged-base", + ), pytest.param( rebase_needed_description(BaseCheck("a" * 40, 300), base_ref="main"), True, id="rebase-needed" ), @@ -304,6 +335,7 @@ def test_status_descriptions_fit_github_limit(): *_STATUS_DESCRIPTIONS.values(), CI_PASSED_REVOKED_DESCRIPTION, ci_passed_description(BaseCheck("d" * 40, 199), base_ref=base_ref), + ci_passed_description(BaseCheck("d" * 40, 199, merged=True), base_ref=base_ref), *( rebase_needed_description(BaseCheck("d" * 40, 9999, overlap), base_ref=base_ref) for overlap in ((), ("a",) * 300, None) @@ -352,3 +384,9 @@ def test_ci_modification_note_mentions_fork_execution_only_when_gated(gated): assert (GATED_NOTE_SENTENCE in comment) is gated assert comment.endswith(FOOTER) + + +def test_ci_passed_description_names_the_merged_base(): + description = ci_passed_description(BaseCheck("a" * 40, 3, merged=True), base_ref="main") + + assert description == "Fork CI passed; merged with main@aaaaaaaaaa, 3 commits behind" diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index 4e2ddd2bf90..1289264902e 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -62,6 +62,7 @@ from airflow_breeze.utils.fork_ci_reconcile import ( BypassFinder, DraftChange, ForkObservation, + ForkRunMerge, ForkState, GateChangeFinder, JobStatusReporter, @@ -90,6 +91,7 @@ from airflow_breeze.utils.fork_ci_reconcile import ( find_project_ci_passed_url, find_project_ci_run_url, find_state_comment, + find_tested_merge, is_cancelled_gate_run, is_gated_skip_run, is_own_pilot_change, @@ -105,6 +107,7 @@ from airflow_breeze.utils.fork_ci_reconcile import ( read_job_status_failure_runs, read_project_ci_passed_url, read_project_ci_run_url, + read_tested_merge, report_project_ci_passed, report_project_ci_runs_started, run_reconcile, @@ -2957,6 +2960,138 @@ def test_compare_with_base(client, responses, expected, expected_calls): assert [call.args[0] for call in client.rest_get.call_args_list] == expected_calls +TESTED_BASE = "b" * 40 +TESTED_MERGE_BASE = "e" * 40 +TESTED = ForkRunMerge("providers-fab/v1-5", TESTED_BASE, TESTED_MERGE_BASE) +TESTED_HEAD_COMPARE = f"repos/{REPO}/compare/{TESTED_BASE}...someone:abc123?per_page=1" +TESTED_BASE_COMPARE = f"repos/{REPO}/compare/{TESTED_BASE}...providers-fab%2Fv1-5?per_page=1" + + +def _tested_base(status="ahead", ahead_by=7): + return {"status": status, "ahead_by": ahead_by} + + [email protected]( + ("tested", "responses", "expected"), + [ + pytest.param( + TESTED, + [_compare(0, [], merge_base=TESTED_MERGE_BASE), _tested_base()], + BaseCheck(TESTED_BASE, 7, (), merged=True), + id="merged", + ), + pytest.param( + TESTED, + [_compare(0, [], merge_base=TESTED_MERGE_BASE), _tested_base("identical", 0)], + BaseCheck(TESTED_BASE, 0, (), merged=True), + id="merged-with-the-tip", + ), + pytest.param( + TESTED._replace(base_ref="main"), + [_compare(5, _files("a.py")), _compare(0, _files("c.py"))], + BaseCheck(MERGE_BASE, 5, ()), + id="merged-with-another-branch", + ), + pytest.param( + TESTED, + [ + _compare(0, [], merge_base="c" * 40), + _tested_base(), + _compare(5, _files("a.py")), + _compare(0, []), + ], + BaseCheck(MERGE_BASE, 5, ()), + id="other-merge-base", + ), + pytest.param( + TESTED, + [ + _compare(0, [], merge_base=TESTED_MERGE_BASE), + _tested_base("diverged"), + _compare(5, _files("a.py")), + _compare(0, []), + ], + BaseCheck(MERGE_BASE, 5, ()), + id="not-on-the-base-branch", + ), + pytest.param( + TESTED, + [GitHubError("gh api failed: HTTP 500"), _compare(5, _files("a.py")), _compare(0, [])], + BaseCheck(MERGE_BASE, 5, ()), + id="merge-not-checkable", + ), + ], +) +def test_compare_with_base_checks_the_merge_the_fork_run_tested(client, tested, responses, expected): + client.rest_get.side_effect = responses + + check = compare_with_base( + client, REPO, _pr(base_ref="providers-fab/v1-5"), find_tested_merge=lambda: tested + ) + + assert check == expected + if expected.merged: + assert [call.args[0] for call in client.rest_get.call_args_list] == [ + TESTED_HEAD_COMPARE, + TESTED_BASE_COMPARE, + ] + + +def _build_info(*steps): + return { + "name": "Build info", + "steps": [{"name": name, "conclusion": conclusion} for name, conclusion in steps], + } + + +MERGE_STEP = ( + f"Test merged with apache/airflow providers-fab/v1-5@{TESTED_BASE}, merge base {TESTED_MERGE_BASE}" +) + + [email protected]( + ("jobs", "expected"), + [ + pytest.param([_build_info(("Checkout", "success"), (MERGE_STEP, "success"))], TESTED, id="named"), + pytest.param([_build_info((MERGE_STEP, "skipped"))], None, id="step-skipped"), + pytest.param( + [_build_info(("Test merged with apache/airflow main@, merge base ", "skipped"))], + None, + id="no-merge", + ), + pytest.param( + [{"name": "Static checks", "steps": [{"name": MERGE_STEP, "conclusion": "success"}]}], + None, + id="other-job", + ), + pytest.param([{"name": "Build info"}], None, id="steps-not-listed"), + ], +) +def test_find_tested_merge(jobs, expected): + assert find_tested_merge(jobs) == expected + + +def test_read_tested_merge_reuses_the_jobs_read_in_this_run(client): + read_jobs = {1: [_build_info((MERGE_STEP, "success"))]} + + assert read_tested_merge(client, _pr(), COMPLETED_RUN, read_jobs) == TESTED + client.get_run_jobs.assert_not_called() + + +def test_read_tested_merge_reads_the_jobs_once(client): + client.get_run_jobs.return_value = [_build_info((MERGE_STEP, "success"))] + read_jobs: dict = {} + + assert read_tested_merge(client, _pr(), COMPLETED_RUN, read_jobs) == TESTED + assert read_jobs == {1: client.get_run_jobs.return_value} + + +def test_read_tested_merge_without_readable_jobs(client): + client.get_run_jobs.side_effect = GitHubError("gh api failed: HTTP 502") + + assert read_tested_merge(client, _pr(), COMPLETED_RUN, {}) is None + + def _fork_job(name, status="completed", conclusion="success", url=None): return { "name": name, @@ -5038,3 +5173,17 @@ def test_monitor_interval_is_the_monitor_workflow_cadence(): crons = [entry["cron"] for entry in workflow[True]["schedule"]] assert crons == [f"*/{MONITOR_INTERVAL // timedelta(minutes=1)} * * * *"] + + [email protected]("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", autospec=True) [email protected]("airflow_breeze.utils.fork_ci_reconcile.classify", autospec=True) +def test_run_reconcile_compares_the_merge_the_fork_run_tested(mock_classify, mock_reporter, client, tmp_path): + mock_classify.return_value = Plan(state=ForkState.RUNNING) + mock_reporter.return_value.report.return_value = None + mock_reporter.return_value.deferred = [] + mock_reporter.return_value.delayed = [] + + _reconcile_with_reporter(client, tmp_path) + + compare_base = mock_classify.call_args.kwargs["compare_base"] + assert compare_base.keywords["find_tested_merge"].func is read_tested_merge diff --git a/scripts/ci/prek/check_fork_ci_tests_merge.py b/scripts/ci/prek/check_fork_ci_tests_merge.py new file mode 100755 index 00000000000..41c999ff4cb --- /dev/null +++ b/scripts/ci/prek/check_fork_ci_tests_merge.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11,<3.12" +# dependencies = [ +# "pyyaml>=6.0.3", +# "rich>=13.6.0", +# ] +# /// +""" +Check that every job of a fork CI run tests the pushed commit merged with the base branch. + +A fork run checks out the pushed commit; the ``merge-with-base`` action then replaces it with its +merge into the base branch, which is what a pull_request run of project CI tests (AIP-120). The +check covers ``ci-amd.yml``, ``ci-arm.yml`` and every workflow they call, directly or not: + +- every call of a reusable workflow passes ``merge-with``, and every reusable workflow takes it; +- in every job that checks out the pushed commit into the workspace root, the merge step follows + the last such checkout. + +A checkout of another ref (a constraints branch, the gate configuration) or into a sub-directory is +not the pull request's sources and is not counted. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any + +import yaml +from rich.console import Console + +console = Console(color_system="standard", width=200) + +WORKFLOWS_DIR = Path(__file__).resolve().parents[3] / ".github" / "workflows" +TOP_WORKFLOWS = ("ci-amd.yml", "ci-arm.yml") +MERGE_ACTION = "./.github/actions/merge-with-base" +REUSABLE_WORKFLOW = re.compile(r"^\./\.github/workflows/([\w.-]+\.yml)$") +# Refs a checkout of the pushed commit is written with. +PUSHED_COMMIT_REFS = frozenset({"", "${{ github.sha }}", "${{ inputs.checkout-ref }}"}) +# Jobs that read the pushed commit itself rather than test it. +EXEMPT_JOBS = { + **{(top, "gate"): "reads the gate configuration" for top in TOP_WORKFLOWS}, + **{ + (top, "build-info"): "selects tests from the pushed commit's changes and resolves the merge base" + for top in TOP_WORKFLOWS + }, + **{(top, "notify-slack"): "only reports on the run" for top in TOP_WORKFLOWS}, + **{(top, "summarize-warnings"): "only reports on the run" for top in TOP_WORKFLOWS}, +} + + +def load_workflow(name: str, workflows_dir: Path) -> dict[str, Any]: + return yaml.safe_load((workflows_dir / name).read_text()) + + +def find_called_workflows(workflow: dict[str, Any]) -> set[str]: + called = set() + for job in (workflow.get("jobs") or {}).values(): + match = REUSABLE_WORKFLOW.match(str(job.get("uses", ""))) + if match: + called.add(match.group(1)) + return called + + +def find_fork_run_workflows(workflows_dir: Path) -> dict[str, dict[str, Any]]: + """The workflows a fork run executes: the top ones and every workflow they call.""" + found: dict[str, dict[str, Any]] = {} + pending = list(TOP_WORKFLOWS) + while pending: + name = pending.pop() + if name in found: + continue + found[name] = load_workflow(name, workflows_dir) + pending.extend(find_called_workflows(found[name])) + return found + + +def is_pushed_commit_checkout(step: dict[str, Any]) -> bool: + if not str(step.get("uses", "")).startswith("actions/checkout@"): + return False + options = step.get("with") or {} + return options.get("path", ".") in (".", "") and str(options.get("ref", "")) in PUSHED_COMMIT_REFS + + +def check_job(workflow_name: str, job_id: str, job: dict[str, Any]) -> list[str]: + if REUSABLE_WORKFLOW.match(str(job.get("uses", ""))): + if "merge-with" not in (job.get("with") or {}): + return [f"job [yellow]{job_id}[/] calls {job['uses']} without passing `merge-with`"] + return [] + if (workflow_name, job_id) in EXEMPT_JOBS: + return [] + steps = job.get("steps") or [] + checkouts = [index for index, step in enumerate(steps) if is_pushed_commit_checkout(step)] + if not checkouts: + return [] + merges = [index for index, step in enumerate(steps) if step.get("uses") == MERGE_ACTION] + if not any(index > checkouts[-1] for index in merges): + return [ + f"job [yellow]{job_id}[/] checks out the pushed commit without a `{MERGE_ACTION}` step " + "after its last such checkout" + ] + return [] + + +def check_workflow(name: str, workflow: dict[Any, Any]) -> list[str]: + errors = [] + if name not in TOP_WORKFLOWS: + # PyYAML reads the bare `on:` key as boolean True. + triggers = workflow.get("on", workflow.get(True)) or {} + call = triggers.get("workflow_call") if isinstance(triggers, dict) else None + if "merge-with" not in ((call or {}).get("inputs") or {}): + errors.append("does not take a `merge-with` input") + for job_id, job in (workflow.get("jobs") or {}).items(): + errors.extend(check_job(name, job_id, job)) + return errors + + +def main(workflows_dir: Path = WORKFLOWS_DIR) -> int: + errors = [ + f"[red].github/workflows/{name}[/]: {error}" + for name, workflow in sorted(find_fork_run_workflows(workflows_dir).items()) + for error in check_workflow(name, workflow) + ] + if not errors: + return 0 + for error in errors: + console.print(error) + console.print( + "\nFork CI tests the pushed commit merged with the base branch, so every job of a fork run must " + f"merge it after checking it out: add a `uses: {MERGE_ACTION}` step taking `merge-with`, pass " + "`merge-with` to called workflows, or add the job to EXEMPT_JOBS in " + "scripts/ci/prek/check_fork_ci_tests_merge.py with the reason it does not test the sources." + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py b/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py new file mode 100644 index 00000000000..f2f90421fc4 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_fork_ci_tests_merge.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +import yaml +from check_fork_ci_tests_merge import MERGE_ACTION, check_job, check_workflow, main + +CHECKOUT = {"uses": "actions/checkout@v7"} +MERGE = {"uses": MERGE_ACTION, "with": {"merge-with": "${{ inputs.merge-with }}"}} + + [email protected]( + ("steps", "valid"), + [ + pytest.param([CHECKOUT, MERGE], True, id="merge-after-checkout"), + pytest.param([CHECKOUT], False, id="no-merge"), + pytest.param([MERGE, CHECKOUT], False, id="merge-before-checkout"), + pytest.param( + [CHECKOUT, MERGE, {"uses": "actions/checkout@v7", "with": {"ref": "${{ github.sha }}"}}], + False, + id="checkout-again-after-merge", + ), + pytest.param( + [{"uses": "actions/checkout@v7", "with": {"ref": "${{ inputs.checkout-ref }}"}}, MERGE], + True, + id="checkout-ref-input", + ), + pytest.param( + [CHECKOUT, MERGE, {"uses": "actions/checkout@v7", "with": {"path": "constraints"}}], + True, + id="checkout-into-sub-directory", + ), + pytest.param( + [{"uses": "actions/checkout@v7", "with": {"ref": "constraints-main"}}], + True, + id="checkout-of-another-ref", + ), + pytest.param([{"run": "echo"}], True, id="no-checkout"), + ], +) +def test_check_job_steps(steps, valid): + assert (check_job("tests.yml", "job", {"steps": steps}) == []) is valid + + [email protected]( + ("job", "valid"), + [ + pytest.param({"uses": "./.github/workflows/x.yml", "with": {"merge-with": "a b"}}, True, id="passes"), + pytest.param({"uses": "./.github/workflows/x.yml", "with": {}}, False, id="missing"), + pytest.param({"uses": "./.github/workflows/x.yml"}, False, id="no-with"), + ], +) +def test_check_job_calls(job, valid): + assert (check_job("tests.yml", "job", job) == []) is valid + + [email protected]("job_id", ["gate", "build-info", "notify-slack", "summarize-warnings"]) +def test_exempt_jobs_of_the_top_workflows(job_id): + assert check_job("ci-amd.yml", job_id, {"steps": [CHECKOUT]}) == [] + assert check_job("other.yml", job_id, {"steps": [CHECKOUT]}) != [] + + [email protected]( + ("inputs", "valid"), + [ + pytest.param({"merge-with": {"type": "string"}}, True, id="takes-it"), + pytest.param({"other": {"type": "string"}}, False, id="lacks-it"), + ], +) +def test_reusable_workflows_take_merge_with(inputs, valid): + workflow = {True: {"workflow_call": {"inputs": inputs}}, "jobs": {}} + assert (check_workflow("reusable.yml", workflow) == []) is valid + + +def _write(path, workflow): + path.write_text(yaml.safe_dump(workflow)) + + +def test_main_follows_called_workflows(tmp_path): + call = {"uses": "./.github/workflows/called.yml", "with": {"merge-with": "a b"}} + for top in ("ci-amd.yml", "ci-arm.yml"): + _write(tmp_path / top, {"on": {"push": {}}, "jobs": {"tests": call}}) + _write( + tmp_path / "called.yml", + {"on": {"workflow_call": {"inputs": {"merge-with": {}}}}, "jobs": {"j": {"steps": [CHECKOUT]}}}, + ) + _write(tmp_path / "unrelated.yml", {"on": {"push": {}}, "jobs": {"j": {"steps": [CHECKOUT]}}}) + + assert main(tmp_path) == 1 + + _write( + tmp_path / "called.yml", + { + "on": {"workflow_call": {"inputs": {"merge-with": {}}}}, + "jobs": {"j": {"steps": [CHECKOUT, MERGE]}}, + }, + ) + + assert main(tmp_path) == 0
