This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-3-test by this push:
new a1918b5b10a Remove pull_request_target workflow
a1918b5b10a is described below
commit a1918b5b10a2ebe9d1789322d68af6709010ceba
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jan 12 15:36:24 2025 +0100
Remove pull_request_target workflow
---
.github/workflows/build-images.yml | 444 -------------------------------------
1 file changed, 444 deletions(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
deleted file mode 100644
index f14d9d7a28a..00000000000
--- a/.github/workflows/build-images.yml
+++ /dev/null
@@ -1,444 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: "Build Images"
-on: # yamllint disable-line rule:truthy
- pull_request_target:
- branches:
- - main
-permissions:
- # all other permissions are set to none
- contents: read
-env:
- MOUNT_SELECTED_LOCAL_SOURCES: "false"
- ANSWER: "yes"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # This token is WRITE one - pull_request_target type of events always have
the WRITE token
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha ||
github.sha }}"
- USE_SUDO: "true"
-
-concurrency:
- group: build-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
- build-info:
- timeout-minutes: 10
- name: "Build Info"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-20.04' }}
- env:
- TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
- outputs:
- runs-on: ${{ github.repository == 'apache/airflow' && '["self-hosted"]'
|| '["ubuntu-20.04"]' }}
- python-versions: "${{ steps.selective-checks.python-versions }}"
- upgrade-to-newer-dependencies: ${{
steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- all-python-versions-list-as-string: >-
- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string
}}
- default-python-version: ${{
steps.selective-checks.outputs.default-python-version }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
- image-build: ${{ steps.selective-checks.outputs.image-build }}
- cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
- default-branch: ${{ steps.selective-checks.outputs.default-branch }}
- default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
- debian-version: ${{ steps.selective-checks.outputs.debian-version }}
- target-commit-sha:
"${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
- github.event.pull_request.head.sha ||
- github.sha
- }}"
- steps:
- - name: Discover PR merge commit
- id: discover-pr-merge-commit
- run: |
- TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}'
--jq .merge_commit_sha)"
- echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV
- echo "::set-output name=target-commit-sha::${TARGET_COMMIT_SHA}"
- if: github.event_name == 'pull_request_target'
- # The labels in the event aren't updated when re-triggering the job, So
lets hit the API to get
- # up-to-date values
- - name: Get latest PR labels
- id: get-latest-pr-labels
- run: |
- echo -n "::set-output name=pull-request-labels::"
- gh api graphql --paginate -F
node_id=${{github.event.pull_request.node_id}} -f query='
- query($node_id: ID!, $endCursor: String) {
- node(id:$node_id) {
- ... on PullRequest {
- labels(first: 100, after: $endCursor) {
- nodes { name }
- pageInfo { hasNextPage endCursor }
- }
- }
- }
- }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]'
- if: github.event_name == 'pull_request_target'
- # Retrieve it to be able to determine which files has changed in the
incoming commit of the PR
- # we checkout the target commit and it's parent to be able to compare
them
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - uses: actions/checkout@v2
- with:
- ref: ${{ env.TARGET_COMMIT_SHA }}
- persist-credentials: false
- fetch-depth: 2
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- # We do not have output from selective checks yet, so we need to
hardcode python
- python-version: 3.7
- cache: 'pip'
- cache-dependency-path: ./dev/breeze/setup*
- - name: "Retrieve defaults from branch_defaults.py"
- # We cannot "execute" the branch_defaults.py python code here because
that would be
- # a security problem (we cannot run any code that comes from the
sources coming from the PR.
- # Therefore, we extract the branches via embedded Python code
- # we need to do it before next step replaces checked-out breeze and
scripts code coming from
- # the PR, because the PR defaults have to be retrieved here.
- id: defaults
- run: |
- python - <<EOF >>$GITHUB_ENV
- from pathlib import Path
- import re
- import sys
-
- DEFAULTS_CONTENT =
Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
- BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
- CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH =
"(.*)"$'
- DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$'
-
- branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT,
re.MULTILINE).group(1)
- constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN,
DEFAULTS_CONTENT, re.MULTILINE).group(1)
- debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT,
re.MULTILINE).group(1)
-
- output = f"""
- DEFAULT_BRANCH={branch}
- DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
- DEBIAN_VERSION={debian_version}
- """.strip()
-
- print(output)
- # Stdout is redirected to GITHUB_ENV but we also print it to stderr
to see it in ci log
- print(output, file=sys.stderr)
- EOF
- - name: Checkout main branch to use breeze from there.
- uses: actions/checkout@v3
- with:
- ref: "main"
- persist-credentials: false
- submodules: recursive
- - run: ./scripts/ci/install_breeze.sh
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: "${{
steps.get-latest-pr-labels.outputs.pull-request-labels }}"
- COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
- run: breeze ci selective-check
- - name: env
- run: printenv
- env:
- PR_LABELS: ${{
steps.get-latest-pr-labels.outputs.pull-request-labels }}
- GITHUB_CONTEXT: ${{ toJson(github) }}
-
- build-ci-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build CI images ${{
needs.build-info.outputs.all-python-versions-list-as-string }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
- needs: [build-info]
- if: |
- needs.build-info.outputs.image-build == 'true' &&
- github.event.pull_request.head.repo.full_name != 'apache/airflow'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
- BACKEND: sqlite
- outputs: ${{toJSON(needs.build-info.outputs) }}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.target-commit-sha }}
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- - name: >
- Checkout "main branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v3
- with:
- path: "main-airflow"
- ref: "main"
- persist-credentials: false
- submodules: recursive
- - name: >
- Override "scripts/ci" with the "main" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rfv "scripts/ci"
- rm -rfv "dev"
- mv -v "main-airflow/scripts/ci" "scripts"
- mv -v "main-airflow/dev" "."
- - uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- cache: 'pip'
- cache-dependency-path: ./dev/breeze/setup*
- - run: ./scripts/ci/install_breeze.sh
- - name: "Free space"
- run: breeze ci free-space
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
- run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
- if: needs.build-info.outputs.default-branch == 'main'
- - name: Compile www assets
- run: breeze compile-www-assets
- - name: >-
- Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- run: breeze ci-image build --push --tag-as-latest --run-in-parallel
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.all-python-versions-list-as-string }}
- - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}
- if: failure() || cancelled()
- run: breeze ci-image build --push --empty-image --run-in-parallel
- env:
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- - name: "Candidates for pip resolver backtrack triggers"
- if: failure() || cancelled()
- run: >
- breeze ci find-newer-dependencies --max-age 1
- --python "${{ needs.build-info.outputs.default-python-version }}"
- - name: "Fix ownership"
- run: breeze ci fix-ownership
- if: always()
-
- build-prod-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build PROD images ${{
needs.build-info.outputs.all-python-versions-list-as-string }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
- needs: [build-info, build-ci-images]
- if: |
- needs.build-info.outputs.image-build == 'true' &&
- github.event.pull_request.head.repo.full_name != 'apache/airflow'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
- BACKEND: sqlite
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.target-commit-sha }}
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- - name: >
- Checkout "main" branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v3
- with:
- path: "main-airflow"
- ref: "main"
- persist-credentials: false
- submodules: recursive
- - name: >
- Override "scripts/ci" with the "main" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rfv "scripts/ci"
- rm -rfv "dev"
- mv -v "main-airflow/scripts/ci" "scripts"
- mv -v "main-airflow/dev" "."
- - uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- cache: 'pip'
- cache-dependency-path: ./dev/breeze/setup*
- - run: ./scripts/ci/install_breeze.sh
- - name: "Free space"
- run: breeze ci free-space
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
- run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
- - name: >
- Pull CI image for PROD build:
- ${{ needs.build-info.outputs.default-python-version }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}
- run: breeze ci-image pull --tag-as-latest
- env:
- # Always use default Python version of CI image for preparing
packages
- PYTHON_MAJOR_MINOR_VERSION: ${{
needs.build-info.outputs.default-python-version }}
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- - name: "Cleanup dist and context file"
- run: rm -fv ./dist/* ./docker-context-files/*
- - name: "Prepare providers packages"
- run: >
- breeze release-management prepare-provider-packages
- --package-list-file ./scripts/ci/installed_providers.txt
- --package-format wheel
- --version-suffix-for-pypi dev0
- if: needs.build-info.outputs.default-branch == 'main'
- - name: "Prepare airflow package"
- run: >
- breeze release-management prepare-airflow-package
- --package-format wheel --version-suffix-for-pypi dev0
- - name: "Move dist packages to docker-context files"
- run: mv -v ./dist/*.whl ./docker-context-files
- - name: >-
- Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- run: >
- breeze prod-image build
- --run-in-parallel
- --tag-as-latest
- --push
- --install-packages-from-context
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.all-python-versions-list-as-string }}
- - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- if: failure() || cancelled()
- run: breeze prod-image build --cleanup-context --push --empty-image
--run-in-parallel
- env:
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- - name: "Fix ownership"
- run: breeze ci fix-ownership
- if: always()
-
-
- build-ci-images-arm:
- timeout-minutes: 50
- name: "Build ARM CI images ${{
needs.build-info.outputs.all-python-versions-list-as-string }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
- needs: [build-info, build-prod-images]
- if: |
- needs.build-info.outputs.image-build == 'true' &&
- needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
- github.event.pull_request.head.repo.full_name != 'apache/airflow'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
- DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
- BACKEND: sqlite
- outputs: ${{toJSON(needs.build-info.outputs) }}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - uses: actions/checkout@v2
- with:
- ref: ${{ needs.build-info.outputs.target-commit-sha }}
- persist-credentials: false
- submodules: recursive
- - name: >
- Checkout "main" branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "main"
- persist-credentials: false
- submodules: recursive
- - name: >
- Override "scripts/ci" with the "main" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rfv "scripts/ci"
- rm -rfv "dev"
- mv -v "main-airflow/scripts/ci" "scripts"
- mv -v "main-airflow/dev" "."
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- - run: ./scripts/ci/install_breeze.sh
- - name: "Free space"
- run: breeze ci free-space
- - name: "Start ARM instance"
- run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- - name: >
- Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- run: >
- breeze ci-image build --run-in-parallel --builder airflow_cache
--platform "linux/arm64"
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.all-python-versions-list-as-string }}
- - name: "Stop ARM instance"
- run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
- - name: "Fix ownership"
- run: breeze ci fix-ownership
- if: always()
