This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new cdfc301630e Remove pull_request_target workflow
cdfc301630e is described below
commit cdfc301630e57d900153bddec0b9687efd53db29
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jan 12 19:39:59 2025 +0100
Remove pull_request_target workflow
---
.github/workflows/build-images-workflow-run.yml | 427 ------------------------
1 file changed, 427 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml
b/.github/workflows/build-images-workflow-run.yml
deleted file mode 100644
index 5c85cb44f07..00000000000
--- a/.github/workflows/build-images-workflow-run.yml
+++ /dev/null
@@ -1,427 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: "Build Images"
-on: # yamllint disable-line rule:truthy
- workflow_run:
- workflows: ["CI Build"]
- types: ['requested']
-env:
- MOUNT_LOCAL_SOURCES: "false"
- FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "true"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- USE_GITHUB_REGISTRY: "true"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false'
}}
-
-jobs:
-
- cancel-workflow-runs:
- timeout-minutes: 10
- name: "Cancel workflow runs"
- runs-on: ubuntu-20.04
- outputs:
- sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- sourceHeadBranch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}
- sourceHeadSha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- mergeCommitSha: ${{ steps.source-run-info.outputs.mergeCommitSha }}
- targetCommitSha: ${{ steps.source-run-info.outputs.targetCommitSha }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- targetBranch: ${{ steps.source-run-info.outputs.targetBranch }}
- sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }}
- cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
- buildImages: ${{ steps.build-images.outputs.buildImages }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Get information about the original trigger of the run"
- uses: ./.github/actions/get-workflow-origin
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- sourceRunId: ${{ github.event.workflow_run.id }}
- - name: "Cancel duplicated 'CI Build' runs"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- sourceRunId: ${{ github.event.workflow_run.id }}
- - name: "Output BUILD_IMAGES"
- id: build-images
- run: |
- # Workaround - jobs cannot access env variable in "ifs"
- #
https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16
- echo "::set-output name=buildImages::${BUILD_IMAGES}"
- - name: "Cancel duplicated 'Build Image' runs"
- # We find duplicates of our own "Build Image" runs - due to a missing
feature
- # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
- # trick ¯\_(ツ)_/¯. We name the build-info job appropriately
- # and then we try to find and cancel all the jobs with the same Event
+ Repo + Branch as the
- # current Event/Repo/Branch combination.
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: namedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- jobNameRegexps: >
- [".*Event: ${{ steps.source-run-info.outputs.sourceEvent }}
- Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}.*"]
- if: env.BUILD_IMAGES == 'true'
- - name: "Cancel all 'CI Build' runs where some jobs failed"
- # We find any of the "CI Build" workflow runs, where any of the
important jobs
- # failed. The important jobs are selected by the regexp array below.
- # We also produce list of canceled "CI Build' runs as output, so that
we
- # can cancel all the matching "Build Images" workflow runs in the two
following steps.
- # Yeah. Adding to the complexity ¯\_(ツ)_/¯.
- uses: ./.github/actions/cancel-workflow-runs
- id: cancel-failed
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: failedJobs
- sourceRunId: ${{ github.event.workflow_run.id }}
- notifyPRCancel: true
- jobNameRegexps: >
- ["^Static checks.*", "^Build docs$", "^Spell check docs$",
"^Backport packages$",
- "^Checks: Helm tests$", "^Test OpenAPI*"]
- - name: "Extract canceled failed runs"
- # We use this step to build regexp that will be used to match the
Source Run id in
- # the build-info job below. If we cancelled some "CI Build" runs in
the "cancel-failed' step
- # above - we want to cancel also the corresponding "Build Images"
runs. Again we have
- # to match the jobs using job name rather than use proper API because
that feature
- # is currently missing in GitHub Actions ¯\_(ツ)_/¯.
- id: extract-cancelled-failed-runs
- if: steps.cancel-failed.outputs.cancelledRuns != '[]'
- run: |
- REGEXP="Source Run id: "
- SEPARATOR=""
- for run_id in $(echo "${{
steps.cancel-failed.outputs.cancelledRuns }}" | jq '.[]')
- do
- REGEXP="${REGEXP}${SEPARATOR}(${run_id})"
- SEPARATOR="|"
- done
- echo "::set-output name=matching-regexp::[\"${REGEXP}\"]"
- - name: "Cancel triggered 'Build Images' runs for the cancelled failed
runs"
- # In case we do have some cancelled jobs in the "cancel-failed" step
above
- # We take the extracted regexp array prepared in the previous step and
we use
- # it to cancel any jobs that have matching names containing Source Run
Id:
- # followed by one of the run ids. Yes I know it's super complex
¯\_(ツ)_/¯.
- if: env.BUILD_IMAGES == 'true' &&
steps.cancel-failed.outputs.cancelledRuns != '[]'
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: namedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- jobNameRegexps: ${{
steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
- - name: "Cancel duplicated 'CodeQL' runs"
- uses: ./.github/actions/cancel-workflow-runs
- id: cancel
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: 'codeql-analysis.yml'
- - name: "Set Docker Cache Directive"
- id: cache-directive
- run: |
- if [[ ${{ steps.source-run-info.outputs.sourceEvent }} == 'schedule'
]]; then
- echo "::set-output name=docker-cache::disabled"
- else
- echo "::set-output name=docker-cache::pulled"
- fi
- - name: "Cancel all duplicated 'Build Image' runs"
- # We find duplicates of all "Build Image" runs - due to a missing
feature
- # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
- # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then
we try to match
- # all the jobs with the same Event + Repo + Branch match and cancel
all the duplicates for those
- # This might cancel own run, so this is the last step in the job
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: allDuplicatedNamedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- selfPreservation: false
- jobNameRegexps: '["Event: \\S* Repo: \\S* Branch: \\S* "]'
-
- build-info:
- # The name is such long because we are using it to cancel duplicated
'Build Images' runs
- # by matching Event/Repo/Branch. This is a workaround for a missing
feature of GitHub
- # Actions to link the source workflow run and the triggered workflow_run
one.
- # We are also cancelling SourceRunId in case we determine that we should
cancel the source
- # Run because of some failing jobs in the source run. Again ¯\_(ツ)_/¯.
- name: >
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha
}}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- runs-on: ubuntu-20.04
- needs: [cancel-workflow-runs]
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- pythonVersions: ${{ steps.selective-checks.python-versions }}
- upgradeToLatestConstraints: ${{
steps.selective-checks.outputs.upgrade-to-latest-constraints }}
- allPythonVersions: ${{
steps.selective-checks.outputs.all-python-versions }}
- defaultPythonVersion: ${{
steps.selective-checks.outputs.default-python-version }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
- image-build: ${{ steps.selective-checks.outputs.image-build }}
- if: >
- needs.cancel-workflow-runs.outputs.buildImages == 'true'
- steps:
- # First fetch the sha of merge commit in case it is pull request so that
we can
- # Run selective tests
- - name: >
- Fetch merge commit ${{ github.ref }} ( ${{ github.sha }}:
- merge_commit ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha
}} )
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- fetch-depth: 2
- if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: >
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{
needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- run: printenv
- - name: >
- Fetch incoming commit ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- fetch-depth: 2
- persist-credentials: false
- if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request'
- # checkout the master version again, to use the right script in master
workflow
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: Selective checks
- id: selective-checks
- env:
- EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- TARGET_COMMIT_SHA: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels
}}
- run: |
- if [[ ${EVENT_NAME} == "pull_request" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
-
- build-images:
- timeout-minutes: 80
- name: "Build ${{matrix.image-type}} images ${{matrix.python-version}}"
- runs-on: ubuntu-20.04
- needs: [build-info, cancel-workflow-runs]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because
workflow_run
- # event is run from master for both master and v1-10-tests
- python-version: ${{
fromJson(needs.build-info.outputs.allPythonVersions) }}
- image-type: [CI, PROD]
- fail-fast: true
- if: >
- needs.build-info.outputs.image-build == 'true' &&
- needs.cancel-workflow-runs.outputs.buildImages == 'true'
- env:
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
- UPGRADE_TO_LATEST_CONSTRAINTS: ${{
needs.build-info.outputs.upgradeToLatestConstraints }}
- DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
- steps:
- - name: >
- Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}]
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{
needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- persist-credentials: false
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security
problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore
we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom
grep/awk/sed commands
- # Also 2.7 and 3.5 versions are not allowed to proceed on master
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH"
scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export
DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >>
$GITHUB_ENV
- if [[ \
- ${DEFAULT_BRANCH} != "master" || \
- ( ${PYTHON_MAJOR_MINOR_VERSION} != "2.7" &&
${PYTHON_MAJOR_MINOR_VERSION} != "3.5" ) \
- ]]; then
- echo "::set-output name=proceed::true"
- else
- echo "::set-output name=proceed::false"
- fi
- - name: >
- Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
- persist-credentials: false
- submodules: recursive
- if: steps.defaults.outputs.proceed == 'true'
- - name: Initiate Github Checks for Building image
- # Use the submodule from main, not the PR branch
- uses: ./main-airflow/.github/actions/checks-action
- id: build-image-check
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Status of image build ${{ matrix.image-type }}: ${{
matrix.python-version }}"
- status: "in_progress"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: ${{ matrix.image-type }}: ${{
matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- if: steps.defaults.outputs.proceed == 'true'
- - name: >
- Override "scripts/ci" with the "${{
needs.cancel-workflow-runs.outputs.targetBranch }}" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rf "scripts/ci"
- mv "main-airflow/scripts/ci" "scripts"
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Build CI images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- # locally built CI image is needed to prepare packages for PROD image
build
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Push CI images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_push_ci_images.sh
- if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed ==
'true'
- - name: "Build PROD images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed ==
'true'
- - name: "Push PROD images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_push_production_images.sh
- if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed ==
'true'
- - name: Update Github Checks for Building image with status
- uses: ./main-airflow/.github/actions/checks-action
- if: always() && steps.defaults.outputs.proceed == 'true'
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- check_id: ${{ steps.build-image-check.outputs.check_id }}
- status: "completed"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- conclusion: ${{ job.status }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: ${{ matrix.image-type }}: ${{
matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
-
- cancel-on-build-cancel:
- name: "Cancel 'CI Build' jobs on build image cancelling."
- runs-on: ubuntu-20.04
- if: cancelled()
- needs: [build-images]
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: self
- notifyPRCancel: true
- notifyPRCancelMessage: "Building image for the PR has been cancelled"
- sourceRunId: ${{ github.event.workflow_run.id }}
-
- cancel-on-build-failure:
- name: "Cancel 'CI Build' jobs on build image failing."
- runs-on: ubuntu-20.04
- if: failure()
- needs: [build-images]
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: self
- notifyPRCancel: true
- notifyPRCancelMessage: |
- Building images for the PR has failed. Follow the the workflow
link to check the reason.
- sourceRunId: ${{ github.event.workflow_run.id }}
