This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 9c98a60 Streamline Build Images workflow using new GitHub Actions
features (#15944)
9c98a60 is described below
commit 9c98a60cdd29f0b005bf3abdbfc42aba419fded8
Author: Ash Berlin-Taylor <[email protected]>
AuthorDate: Tue May 25 15:38:42 2021 +0100
Streamline Build Images workflow using new GitHub Actions features (#15944)
Use `pull_request_target` event for building images, and `concurrency` to
automatically cancel old jobs for PRs.
This means that:
- GitHub will automatically cancel old jobs for us, so we don't have to
handle that ourselves (removes most of the use of the
cancel-workflow-action)
- GitHub displays these checks directly on the PR, but it is still run
in the context of our repo, meaning it has access write to our
repo/access to secrets etc.
- Since it shows up directly on the PR checks, we don't need to create the
check in the "CI" workflow to show the status of the Image Build.
- We also don't need to post the comment saying _why_ it failed, as the
Build Image status will show up directly there
- Since `pull_request_target` has information about the PR in the
`github.event` context, we don't need the complex mechanism to find
the "other" PR, we can do a fairly simple API request and filter by
the commit SHA to find and cancel to CI workflow job. (This removes
the final use of the cancel-workflow-action)
One change I had to make here what tag we use for Docker images we build
and push up. Previously we used the "source run ID" (i.e. the id of the
CI run) but with pull_request_target we don't have that anymore. We
could use the same API mechanism we do to cancel to find the target job,
but the only requirement here is for an ID that both jobs know -- the
SHA of the PR branch fills that need
Extra side benefits of this:
- The sidebar of commits in main branch aren't "polluted" with Build
Images for PRs like they were previously.
---
.github/actions/cancel-workflow-runs | 1 -
.github/workflows/build-images-workflow-run.yml | 561 -----------------------
.github/workflows/build-images.yml | 337 ++++++++++++++
.github/workflows/ci.yml | 15 +-
.github/workflows/codeql-analysis.yml | 4 +
.gitmodules | 3 -
BREEZE.rst | 60 +--
CI.rst | 78 ++--
IMAGES.rst | 24 +-
breeze | 44 +-
images/ci/pull_request_ci_flow.md5 | 2 +-
images/ci/pull_request_ci_flow.mermaid | 34 +-
images/ci/pull_request_ci_flow.png | Bin 191467 -> 148327 bytes
images/ci/push_ci_flow.md5 | 2 +-
images/ci/push_ci_flow.mermaid | 33 +-
images/ci/push_ci_flow.png | Bin 205098 -> 160006 bytes
scripts/ci/libraries/_build_images.sh | 47 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 12 +-
18 files changed, 475 insertions(+), 782 deletions(-)
diff --git a/.github/actions/cancel-workflow-runs
b/.github/actions/cancel-workflow-runs
deleted file mode 160000
index 953e057..0000000
--- a/.github/actions/cancel-workflow-runs
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 953e057dc81d3458935a18d1184c386b0f6b5738
diff --git a/.github/workflows/build-images-workflow-run.yml
b/.github/workflows/build-images-workflow-run.yml
deleted file mode 100644
index 1f4f4d5..0000000
--- a/.github/workflows/build-images-workflow-run.yml
+++ /dev/null
@@ -1,561 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: "Build Images"
-on: # yamllint disable-line rule:truthy
- workflow_run:
- workflows: ["CI Build"]
- types: ['requested']
-env:
- MOUNT_SELECTED_LOCAL_SOURCES: "false"
- FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "false"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- USE_GITHUB_REGISTRY: "true"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # This token is WRITE one - workflow_run type of events always have the
WRITE token
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- # This token should not be empty in workflow_run type of event.
- CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false'
}}
- INSTALL_PROVIDERS_FROM_SOURCES: "true"
- GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
-
-jobs:
-
- cancel-workflow-runs:
- timeout-minutes: 10
- name: "Cancel workflow runs"
- runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-20.04' }}
- outputs:
- sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- sourceHeadBranch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}
- sourceHeadSha: ${{ steps.source-run-info.outputs.sourceHeadSha }}
- mergeCommitSha: ${{ steps.source-run-info.outputs.mergeCommitSha }}
- targetCommitSha: ${{ steps.source-run-info.outputs.targetCommitSha }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- targetBranch: ${{ steps.source-run-info.outputs.targetBranch }}
- sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }}
- cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
- buildImages: ${{ steps.build-images.outputs.buildImages }}
- runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]'
|| '["ubuntu-20.04"]' }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Get information about the original trigger of the run"
- uses: ./.github/actions/get-workflow-origin
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- sourceRunId: ${{ github.event.workflow_run.id }}
- - name: "Cancel duplicated 'CI Build' runs"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- sourceRunId: ${{ github.event.workflow_run.id }}
- - name: "Output BUILD_IMAGES"
- id: build-images
- run: |
- # Workaround - jobs cannot access env variable in "ifs"
- #
https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16
- echo "::set-output name=buildImages::${BUILD_IMAGES}"
- - name: "Cancel duplicated 'Build Image' runs"
- # We find duplicates of our own "Build Image" runs - due to a missing
feature
- # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
- # trick ¯\_(ツ)_/¯. We name the build-info job appropriately
- # and then we try to find and cancel all the jobs with the same Event
+ Repo + Branch as the
- # current Event/Repo/Branch combination.
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: namedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- jobNameRegexps: >
- [".*Event: ${{ steps.source-run-info.outputs.sourceEvent }}
- Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}.*"]
- if: env.BUILD_IMAGES == 'true'
- - name: "Cancel all 'CI Build' runs where some jobs failed"
- # We find any of the "CI Build" workflow runs, where any of the
important jobs
- # failed. The important jobs are selected by the regexp array below.
- # We also produce list of canceled "CI Build' runs as output, so that
we
- # can cancel all the matching "Build Images" workflow runs in the two
following steps.
- # Yeah. Adding to the complexity ¯\_(ツ)_/¯.
- uses: ./.github/actions/cancel-workflow-runs
- id: cancel-failed
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: failedJobs
- sourceRunId: ${{ github.event.workflow_run.id }}
- notifyPRCancel: true
- jobNameRegexps: >
- ["^Pylint$", "^Static checks", "^Build docs$", "^Spell check
docs$",
- "^Provider packages", "^Checks: Helm tests$", "^Test OpenAPI*"]
- - name: "Extract canceled failed runs"
- # We use this step to build regexp that will be used to match the
Source Run id in
- # the build-info job below. If we cancelled some "CI Build" runs in
the "cancel-failed' step
- # above - we want to cancel also the corresponding "Build Images"
runs. Again we have
- # to match the jobs using job name rather than use proper API because
that feature
- # is currently missing in GitHub Actions ¯\_(ツ)_/¯.
- id: extract-cancelled-failed-runs
- if: steps.cancel-failed.outputs.cancelledRuns != '[]'
- run: |
- REGEXP="Source Run id: "
- SEPARATOR=""
- for run_id in $(echo "${{
steps.cancel-failed.outputs.cancelledRuns }}" | jq '.[]')
- do
- REGEXP="${REGEXP}${SEPARATOR}(${run_id})"
- SEPARATOR="|"
- done
- echo "::set-output name=matching-regexp::[\"${REGEXP}\"]"
- - name: "Cancel triggered 'Build Images' runs for the cancelled failed
runs"
- # In case we do have some cancelled jobs in the "cancel-failed" step
above
- # We take the extracted regexp array prepared in the previous step and
we use
- # it to cancel any jobs that have matching names containing Source Run
Id:
- # followed by one of the run ids. Yes I know it's super complex
¯\_(ツ)_/¯.
- if: env.BUILD_IMAGES == 'true' &&
steps.cancel-failed.outputs.cancelledRuns != '[]'
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: namedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- jobNameRegexps: ${{
steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
- - name: "Cancel duplicated 'CodeQL' runs"
- uses: ./.github/actions/cancel-workflow-runs
- id: cancel
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: 'codeql-analysis.yml'
- - name: "Set Docker Cache Directive"
- id: cache-directive
- run: |
- if [[ ${{ steps.source-run-info.outputs.sourceEvent }} == 'schedule'
]]; then
- echo "::set-output name=docker-cache::disabled"
- else
- echo "::set-output name=docker-cache::pulled"
- fi
- - name: "Cancel all duplicated 'Build Image' runs"
- # We find duplicates of all "Build Image" runs - due to a missing
feature
- # in GitHub Actions, we have to use Job names to match
Event/Repo/Branch matching
- # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then
we try to match
- # all the jobs with the same Event + Repo + Branch match and cancel
all the duplicates for those
- # This might cancel own run, so this is the last step in the job
- uses: ./.github/actions/cancel-workflow-runs
- with:
- cancelMode: allDuplicatedNamedJobs
- token: ${{ secrets.GITHUB_TOKEN }}
- notifyPRCancel: true
- selfPreservation: false
- jobNameRegexps: '["Event: \\S* Repo: \\S* Branch: \\S* "]'
-
- build-info:
- # The name is such long because we are using it to cancel duplicated
'Build Images' runs
- # by matching Event/Repo/Branch. This is a workaround for a missing
feature of GitHub
- # Actions to link the source workflow run and the triggered workflow_run
one.
- # We are also cancelling SourceRunId in case we determine that we should
cancel the source
- # Run because of some failing jobs in the source run. Again ¯\_(ツ)_/¯.
- name: >
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha
}}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }}
- needs: [cancel-workflow-runs]
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- pythonVersions: ${{ steps.selective-checks.python-versions }}
- upgradeToNewerDependencies: ${{
steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- allPythonVersions: ${{
steps.selective-checks.outputs.all-python-versions }}
- defaultPythonVersion: ${{
steps.selective-checks.outputs.default-python-version }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
- image-build: ${{ steps.selective-checks.outputs.image-build }}
- if: >
- needs.cancel-workflow-runs.outputs.buildImages == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: >
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{
needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- run: printenv
- - name: >
- Fetch incoming commit ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- fetch-depth: 2
- persist-credentials: false
- submodules: recursive
- if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request'
- # checkout the master version again, to use the right script in master
workflow
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: Selective checks
- id: selective-checks
- env:
- EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- TARGET_COMMIT_SHA: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels
}}
- run: |
- if [[ ${EVENT_NAME} == "pull_request" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
-
- build-ci-images:
- timeout-minutes: 80
- name: "Build CI images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }}
- needs: [build-info, cancel-workflow-runs]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because
workflow_run
- # event is run from master for both master and v1-10-tests
- python-version: ${{
fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: >
- needs.build-info.outputs.image-build == 'true' &&
- needs.cancel-workflow-runs.outputs.buildImages == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
- CONTINUE_ON_PIP_CHECK_FAILURE: "true"
- DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
- FORCE_PULL_BASE_PYTHON_IMAGE: >
- ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' ||
'false' }}
- steps:
- - name: >
- Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}]
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{
needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security
problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore
we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom
grep/awk/sed commands
- # Also 2.7 and 3.5 versions are not allowed to proceed on master
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH"
scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export
DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >>
$GITHUB_ENV
- if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \
- ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" ||
${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \
- ]]; then
- echo "::set-output name=proceed::false"
- else
- echo "::set-output name=proceed::true"
- fi
- - name: >
- Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
- persist-credentials: false
- submodules: recursive
- if: steps.defaults.outputs.proceed == 'true'
- - name: Initiate GitHub Checks for Building image
- # Use the submodule from main, not the PR branch
- uses: ./main-airflow/.github/actions/checks-action
- id: build-image-check
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Status of image build CI: ${{ matrix.python-version }}"
- status: "in_progress"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: CI: ${{ matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: steps.defaults.outputs.proceed == 'true'
- - name: >
- Override "scripts/ci" with the "${{
needs.cancel-workflow-runs.outputs.targetBranch }}" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rf "scripts/ci"
- mv "main-airflow/scripts/ci" "scripts"
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Build CI images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Push CI images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_push_ci_images.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: Update GitHub Checks for Building image with status
- uses: ./main-airflow/.github/actions/checks-action
- if: always() && steps.defaults.outputs.proceed == 'true'
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- check_id: ${{ steps.build-image-check.outputs.check_id }}
- status: "completed"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- conclusion: ${{ job.status }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: CI: ${{ matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
-
- build-prod-images:
- timeout-minutes: 80
- name: "Build PROD images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }}
- needs: [build-info, cancel-workflow-runs, build-ci-images]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because
workflow_run
- # event is run from master for both master and v1-10-tests
- python-version: ${{
fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: >
- needs.build-info.outputs.image-build == 'true' &&
- needs.cancel-workflow-runs.outputs.buildImages == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: ${{ github.event.workflow_run.id }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
- FORCE_PULL_BASE_PYTHON_IMAGE: >
- ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' ||
'false' }}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- steps:
- - name: >
- Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}]
- Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
- Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }}
- Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }}
- Run id: ${{ github.run_id }}
- Source Run id: ${{ github.event.workflow_run.id }}
- Sha: ${{ github.sha }}
- Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- Merge commit Sha: ${{
needs.cancel-workflow-runs.outputs.mergeCommitSha }}
- Target commit Sha: ${{
needs.cancel-workflow-runs.outputs.targetCommitSha }}
- uses: actions/checkout@v2
- with:
- ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security
problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore
we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom
grep/awk/sed commands
- # Also 2.7 and 3.5 versions are not allowed to proceed on master
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH"
scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export
DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >>
$GITHUB_ENV
- if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \
- ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" ||
${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \
- ]]; then
- echo "::set-output name=proceed::false"
- else
- echo "::set-output name=proceed::true"
- fi
- - name: >
- Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
- persist-credentials: false
- submodules: recursive
- if: steps.defaults.outputs.proceed == 'true'
- - name: Initiate GitHub Checks for Building image with status
- uses: ./main-airflow/.github/actions/checks-action
- id: build-image-check
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Status of image build PROD: ${{ matrix.python-version }}"
- status: "in_progress"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: PROD: ${{ matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: steps.defaults.outputs.proceed == 'true'
- - name: >
- Override "scripts/ci" with the "${{
needs.cancel-workflow-runs.outputs.targetBranch }}" branch
- so that the PR does not override it
- # We should not override those scripts which become part of the image
as they will not be
- # changed in the image built - we should only override those that are
executed to build
- # the image.
- run: |
- rm -rf "scripts/ci"
- mv "main-airflow/scripts/ci" "scripts"
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Build CI images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- # Pull images built in the previous step
- if: steps.defaults.outputs.proceed == 'true'
- env:
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
- - name: "Build PROD images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: "Push PROD images ${{ matrix.python-version }}:${{
github.event.workflow_run.id }}"
- run: ./scripts/ci/images/ci_push_production_images.sh
- if: steps.defaults.outputs.proceed == 'true'
- - name: Update GitHub Checks for Building image with status
- uses: ./main-airflow/.github/actions/checks-action
- if: always() && steps.defaults.outputs.proceed == 'true'
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- check_id: ${{ steps.build-image-check.outputs.check_id }}
- status: "completed"
- sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
- conclusion: ${{ job.status }}
- details_url: https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }}
- output: >
- {"summary":
- "Building the image: PROD: ${{ matrix.python-version }}. See the
- [Image Build](https://github.com/${{ github.repository
}}/actions/runs/${{ github.run_id }})
- for details" }
-
- cancel-on-build-cancel:
- name: "Cancel 'CI Build' jobs on build image cancelling."
- runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }}
- if: cancelled()
- needs: [cancel-workflow-runs, build-ci-images, build-prod-images]
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: self
- notifyPRCancel: true
- notifyPRCancelMessage: "Building image for the PR has been cancelled"
- sourceRunId: ${{ github.event.workflow_run.id }}
-
- cancel-on-build-failure:
- name: "Cancel 'CI Build' jobs on build image failing."
- runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }}
- if: failure()
- needs: [cancel-workflow-runs, build-ci-images, build-prod-images]
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: ./.github/actions/cancel-workflow-runs
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: self
- notifyPRCancel: true
- notifyPRCancelMessage: |
- Building images for the PR has failed. Follow the workflow link to
check the reason.
- sourceRunId: ${{ github.event.workflow_run.id }}
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
new file mode 100644
index 0000000..cf71c83
--- /dev/null
+++ b/.github/workflows/build-images.yml
@@ -0,0 +1,337 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Build Images"
+on: # yamllint disable-line rule:truthy
+ schedule:
+ - cron: '28 0 * * *'
+ pull_request_target:
+ push:
+ branches: ['main', 'master', 'v1-10-test', 'v1-10-stable', 'v2-0-test']
+env:
+ MOUNT_SELECTED_LOCAL_SOURCES: "false"
+ FORCE_ANSWER_TO_QUESTIONS: "yes"
+ FORCE_PULL_IMAGES: "false"
+ CHECK_IMAGE_FOR_REBUILD: "true"
+ SKIP_CHECK_REMOTE_IMAGE: "true"
+ DB_RESET: "true"
+ VERBOSE: "true"
+ USE_GITHUB_REGISTRY: "true"
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_USERNAME: ${{ github.actor }}
+ # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
+ # Airflow one is going to be used
+ CONSTRAINTS_GITHUB_REPOSITORY: >-
+ ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
+ secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }}
+ # This token is WRITE one - pull_request_target type of events always have
the WRITE token
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # This token should not be empty in pull_request_target type of event.
+ CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }}
+ GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
+ GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
+ INSTALL_PROVIDERS_FROM_SOURCES: "true"
+ GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
+ TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+concurrency:
+ group: build-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+
+ build-info:
+ timeout-minutes: 10
+ name: "Build Info"
+ runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-20.04' }}
+ env:
+ GITHUB_CONTEXT: ${{ toJson(github) }}
+ targetBranch: ${{ github.event.pull_request.base.ref }}
+ pullRequestLabels: "${{ toJSON(github.event.pull_request.labels.*.name)
}}"
+ outputs:
+ runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]'
|| '["ubuntu-20.04"]' }}
+ pythonVersions: "${{ steps.selective-checks.python-versions }}"
+ upgradeToNewerDependencies: ${{
steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
+ allPythonVersions: ${{
steps.selective-checks.outputs.all-python-versions }}
+ defaultPythonVersion: ${{
steps.selective-checks.outputs.default-python-version }}
+ run-tests: ${{ steps.selective-checks.outputs.run-tests }}
+ run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
+ image-build: ${{ steps.dynamic-outputs.outputs.image-build }}
+ cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
+ targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
+ - name: env
+ run: printenv
+ env:
+ dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }}
+ - name: Selective checks
+ id: selective-checks
+ env:
+ PR_LABELS: ${{ env.pullRequestLabels }}
+ run: |
+ if [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then
+ # Run selective checks
+ ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
+ else
+ # Run all checks
+ ./scripts/ci/selective_ci_checks.sh
+ fi
+ - name: Compute dynamic outputs
+ id: dynamic-outputs
+ run: |
+ set -x
+ if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
+ echo "::set-output name=targetBranch::${targetBranch}"
+ else
+ # Direct push to branch, or scheduled build
+ echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}"
+ fi
+
+ if [[ "${{ github.event_name }}" == 'schedule' ]]; then
+ echo "::set-output name=cacheDirective::disabled"
+ else
+ echo "::set-output name=cacheDirective::pulled"
+ fi
+
+ if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" &&
"$BUILD_IMAGES_OVERRIDE" != "false" ]]; then
+ echo "::set-output name=image-build::true"
+ else
+ echo "::set-output name=image-build::false"
+ fi
+ env:
+ SELECTIVE_CHECKS_IMAGE_BUILD: ${{
steps.selective-checks.outputs.image-build }}
+ BUILD_IMAGES_OVERRIDE: ${{
secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE }}
+
+ build-ci-images:
+ timeout-minutes: 80
+ name: "Build CI images ${{matrix.python-version}}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ needs: [build-info]
+ strategy:
+ matrix:
+ # We need to attempt to build all possible versions here because
pull_request_target
+ # event is run from master for both master and v1-10-tests
+ python-version: ${{
fromJson(needs.build-info.outputs.allPythonVersions) }}
+ fail-fast: true
+ if: needs.build-info.outputs.image-build == 'true'
+ env:
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ BACKEND: postgres
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
+ CONTINUE_ON_PIP_CHECK_FAILURE: "true"
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ FORCE_PULL_BASE_PYTHON_IMAGE: >
+ ${{ github.event_name == 'schedule' && 'true' || 'false' }}
+ outputs: ${{toJSON(needs.build-info.outputs) }}
+ steps:
+ - name: Set envs
+ # Can't refer to "global" env to set a var in a job's env context
+ run: |
+ echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >>
"$GITHUB_ENV"
+ - uses: actions/checkout@v2
+ with:
+ ref: ${{ env.TARGET_COMMIT_SHA }}
+ persist-credentials: false
+ submodules: recursive
+ - name: "Retrieve DEFAULTS from the _initialization.sh"
+ # We cannot "source" the script here because that would be a security
problem (we cannot run
+ # any code that comes from the sources coming from the PR. Therefore
we extract the
+ # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom
grep/awk/sed commands
+ # Also 2.7 and 3.5 versions are not allowed to proceed on master
+ id: defaults
+ run: |
+ DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH"
scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+ DEFAULT_CONSTRAINTS_BRANCH=$(grep "export
DEFAULT_CONSTRAINTS_BRANCH" \
+ scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >>
$GITHUB_ENV
+ if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \
+ ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" ||
${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \
+ ]]; then
+ echo "::set-output name=proceed::false"
+ else
+ echo "::set-output name=proceed::true"
+ fi
+ - name: >
+ Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to
'main-airflow' folder
+ to use ci/scripts from there.
+ uses: actions/checkout@v2
+ with:
+ path: "main-airflow"
+ ref: "${{ needs.build-info.outputs.targetBranch }}"
+ persist-credentials: false
+ submodules: recursive
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Setup python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: >
+ Override "scripts/ci" with the "${{
needs.build-info.outputs.targetBranch }}" branch
+ so that the PR does not override it
+ # We should not override those scripts which become part of the image
as they will not be
+ # changed in the image built - we should only override those that are
executed to build
+ # the image.
+ run: |
+ rm -rf "scripts/ci"
+ mv "main-airflow/scripts/ci" "scripts"
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Build CI images ${{ matrix.python-version }}:${{
env.TARGET_COMMIT_SHA }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Push CI images ${{ matrix.python-version }}:${{
env.TARGET_COMMIT_SHA }}"
+ run: ./scripts/ci/images/ci_push_ci_images.sh
+ if: steps.defaults.outputs.proceed == 'true'
+
+ build-prod-images:
+ timeout-minutes: 80
+ name: "Build PROD images ${{matrix.python-version}}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ needs: [build-info, build-ci-images]
+ strategy:
+ matrix:
+ # We need to attempt to build all possible versions here because
pull_request_target
+ # event is run from master for both master and v1-10-tests
+ python-version: ${{
fromJson(needs.build-info.outputs.allPythonVersions) }}
+ fail-fast: true
+ if: needs.build-info.outputs.image-build == 'true'
+ env:
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ BACKEND: postgres
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{
needs.build-info.outputs.upgradeToNewerDependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ FORCE_PULL_BASE_PYTHON_IMAGE: >
+ ${{ github.event_name == 'schedule' && 'true' || 'false' }}
+ VERSION_SUFFIX_FOR_PYPI: ".dev0"
+ steps:
+ - name: Set envs
+ run: |
+ echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >>
"$GITHUB_ENV"
+ echo "GITHUB_REGISTRY_PULL_IMAGE_TAG=${TARGET_COMMIT_SHA}" >>
"$GITHUB_ENV"
+ - uses: actions/checkout@v2
+ with:
+ ref: ${{ env.TARGET_COMMIT_SHA }}
+ persist-credentials: false
+ submodules: recursive
+ - name: "Retrieve DEFAULTS from the _initialization.sh"
+ # We cannot "source" the script here because that would be a security
problem (we cannot run
+ # any code that comes from the sources coming from the PR. Therefore
we extract the
+ # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom
grep/awk/sed commands
+ # Also 2.7 and 3.5 versions are not allowed to proceed on master
+ id: defaults
+ run: |
+ DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH"
scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+ DEFAULT_CONSTRAINTS_BRANCH=$(grep "export
DEFAULT_CONSTRAINTS_BRANCH" \
+ scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >>
$GITHUB_ENV
+ if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \
+ ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" ||
${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \
+ ]]; then
+ echo "::set-output name=proceed::false"
+ else
+ echo "::set-output name=proceed::true"
+ fi
+ - name: >
+ Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to
'main-airflow' folder
+ to use ci/scripts from there.
+ uses: actions/checkout@v2
+ with:
+ path: "main-airflow"
+ ref: "${{ needs.build-info.outputs.targetBranch }}"
+ persist-credentials: false
+ submodules: recursive
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Setup python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: >
+ Override "scripts/ci" with the "${{
needs.build-info.outputs.targetBranch }}" branch
+ so that the PR does not override it
+ # We should not override those scripts which become part of the image
as they will not be
+ # changed in the image built - we should only override those that are
executed to build
+ # the image.
+ run: |
+ rm -rf "scripts/ci"
+ mv "main-airflow/scripts/ci" "scripts"
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Build CI images ${{ matrix.python-version }}:${{
env.TARGET_COMMIT_SHA }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ # Pull images built in the previous step
+ if: steps.defaults.outputs.proceed == 'true'
+ env:
+ GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
+ - name: "Build PROD images ${{ matrix.python-version }}:${{
env.TARGET_COMMIT_SHA }}"
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+ if: steps.defaults.outputs.proceed == 'true'
+ - name: "Push PROD images ${{ matrix.python-version }}:${{
env.TARGET_COMMIT_SHA }}"
+ run: ./scripts/ci/images/ci_push_production_images.sh
+ if: steps.defaults.outputs.proceed == 'true'
+
+ cancel-on-ci-build:
+ name: "Cancel 'CI Build' jobs on workflow failed/cancelled"
+ runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' ||
'ubuntu-20.04' }}
+ if: failure() || cancelled()
+ needs: [build-ci-images, build-prod-images]
+ env:
+ branch: ${{ github.event.pull_request.head.ref }}
+ thisRun: ${{ github.run_id }}
+ steps:
+ - name: Find running CI Build jobs for ${{ env.TARGET_COMMIT_SHA }}
+ run: |
+ if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
+ event_filter="event=pull_request&"
+ else
+ branch="${GITHUB_REF#refs/heads/}"
+ event_filter=""
+ fi
+
+ for cancel_url in $(
+ gh api
"/repos/$GITHUB_REPOSITORY/actions/runs?${event_filter}branch=${branch}" \
+ jq -r '
+ .workflow_runs[] |
+ select(.head_sha == $ENV.TARGET_COMMIT_SHA and .status !=
"completed") |
+ .cancel_url
+ ' \
+ ); do
+ # One of these URls will be _this_ workflow, so lets exclude that!
+ [[ $cancel_url == */$thisRun/* ]] && continue
+
+ echo "Cancelling $cancel_url"
+ gh api -X POST --silent "$cancel_url"
+ done
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c097be7..b026840 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,7 +49,7 @@ env:
# In builds from forks, this token is empty, and this is good because such
builds do not even try
# to push images to the registry.
CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}"
+ GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.event.pull_request.head.sha ||
github.sha }}"
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
INSTALL_PROVIDERS_FROM_SOURCES: "true"
@@ -68,6 +68,10 @@ env:
# by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set
to "false"
GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{
secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
+concurrency:
+ group: ci-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
jobs:
build-info:
@@ -185,14 +189,6 @@ jobs:
id: source-run-info
with:
token: ${{ secrets.GITHUB_TOKEN }}
- - name: >
- Event: ${{ github.event_name }}
- Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- Branch: ${{ github.head_ref }}
- Run id: ${{ github.run_id }}
- Sha: ${{ github.sha }}
- Ref: ${{ github.ref }}
- run: printenv
- name: Set wait for image
id: wait-for-image
run: |
@@ -413,6 +409,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{
github.sha }}"
env:
VERBOSE: false
+ - run: sleep 600
static-checks-pylint:
diff --git a/.github/workflows/codeql-analysis.yml
b/.github/workflows/codeql-analysis.yml
index 0c35b148..5c4b0af 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy
schedule:
- cron: '0 2 * * *'
+concurrency:
+ group: codeql-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
jobs:
selective-checks:
name: Selective checks
diff --git a/.gitmodules b/.gitmodules
index 1779ddd..e03978e 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,6 @@
[submodule ".github/actions/get-workflow-origin"]
path = .github/actions/get-workflow-origin
url = https://github.com/potiuk/get-workflow-origin
-[submodule ".github/actions/cancel-workflow-runs"]
- path = .github/actions/cancel-workflow-runs
- url = https://github.com/potiuk/cancel-workflow-runs
[submodule ".github/actions/checks-action"]
path = .github/actions/checks-action
url = https://github.com/LouisBrunner/checks-action
diff --git a/BREEZE.rst b/BREEZE.rst
index f4044b2..27658cb 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1199,20 +1199,15 @@ This is the current syntax for `./breeze <./breeze>`_:
'breeze shell -- -c "ls -la"'
'breeze -- -c "ls -la"'
- For DockerHub pull --dockerhub-user and --dockerhub-repo flags can be
used to specify
+ For DockerHub pull: --dockerhub-user and --dockerhub-repo flags can be
used to specify
the repository to pull from. For GitHub repository, the
--github-repository
- flag can be used for the same purpose. You can also use
- --github-image-id <COMMIT_SHA>|<RUN_ID> in case you want to pull the
image
- with specific COMMIT_SHA tag or RUN_ID.
+ flag can be used for the same purpose. You can also use
--github-image-id <COMMIT_SHA> in case
+ you want to pull the image with specific COMMIT_SHA tag.
'breeze shell \
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' -
pull/use image with SHA
'breeze \
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' -
pull/use image with SHA
- 'breeze shell \
- --github-image-id 209845560' - pull/use image with RUN_ID
- 'breeze \
- --github-image-id 209845560' - pull/use image with RUN_ID
Most flags are applicable to the shell command as it will run build when
needed.
@@ -1257,12 +1252,9 @@ This is the current syntax for `./breeze <./breeze>`_:
You can also pass '--production-image' flag to build production image
rather than CI image.
- For DockerHub pull. '--dockerhub-user' and '--dockerhub-repo' flags
can be used to specify
- the repository to pull from. For GitHub repository, the
'--github-repository'
- flag can be used for the same purpose. You can also use
- '--github-image-id <COMMIT_SHA>|<RUN_ID>' in case you want to pull the
image with
- specific COMMIT_SHA tag or RUN_ID.
-
+ For DockerHub pulling of base images: '--dockerhub-user' and
'--dockerhub-repo' flags can be
+ used to specify the repository to pull from. For GitHub repository,
the '--github-repository'
+ flag can be used for the same purpose.
Flags:
-p, --python PYTHON_MAJOR_MINOR_VERSION
@@ -1468,18 +1460,6 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --use-github-registry flag is
enabled.
- -s, --github-image-id COMMIT_SHA|RUN_ID
- <RUN_ID> or <COMMIT_SHA> of the image. Images in GitHub registry are
stored with those
- to be able to easily find the image for particular CI runs. Once you
know the
- <RUN_ID> or <COMMIT_SHA>, you can specify it in github-image-id flag
and Breeze will
- automatically pull and use that image so that you can easily
reproduce a problem
- that occurred in CI.
-
- If you use this flag, automatically --use-github-registry is enabled.
-
-
- Default: latest.
-
-v, --verbose
Show verbose information about executed docker, kind, kubectl, helm
commands. Useful for
debugging - when you run breeze with --verbose flags you will be
able to see the commands
@@ -1605,11 +1585,11 @@ This is the current syntax for `./breeze <./breeze>`_:
Pushes images to docker registry. You can push the images to DockerHub
registry (default)
or to the GitHub registry (if --use-github-registry flag is used).
- For DockerHub pushes --dockerhub-user and --dockerhub-repo flags can
be used to specify
+ For DockerHub pushes: --dockerhub-user and --dockerhub-repo flags can
be used to specify
the repository to push to. For GitHub repository, the
--github-repository
flag can be used for the same purpose. You can also add
- --github-image-id <COMMIT_SHA>|<RUN_ID> in case you want to push image
with specific
- SHA tag or run id. In case you specify --github-repository or
--github-image-id, you
+ --github-image-id <COMMIT_SHA> in case you want to push image with
specific
+ SHA tag. In case you specify --github-repository or --github-image-id,
you
do not need to specify --use-github-registry flag.
You can also add --production-image flag to switch to production image
(default is CI one)
@@ -1624,8 +1604,6 @@ This is the current syntax for `./breeze <./breeze>`_:
--github-repository user/airflow' - to push to your user's fork
'breeze push-image \
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - to
push with COMMIT_SHA
- 'breeze push-image \
- --github-image-id 209845560' - to push with RUN_ID
Flags:
@@ -1656,16 +1634,18 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --use-github-registry flag is
enabled.
- -s, --github-image-id COMMIT_SHA|RUN_ID
- <RUN_ID> or <COMMIT_SHA> of the image. Images in GitHub registry are
stored with those
+
+
+
+ -s, --github-image-id COMMIT_SHA
+ <COMMIT_SHA> of the image. Images in GitHub registry are stored with
those
to be able to easily find the image for particular CI runs. Once you
know the
- <RUN_ID> or <COMMIT_SHA>, you can specify it in github-image-id flag
and Breeze will
+ <COMMIT_SHA>, you can specify it in github-image-id flag and Breeze
will
automatically pull and use that image so that you can easily
reproduce a problem
that occurred in CI.
If you use this flag, automatically --use-github-registry is enabled.
-
Default: latest.
-v, --verbose
@@ -2719,16 +2699,18 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --use-github-registry flag is
enabled.
- -s, --github-image-id COMMIT_SHA|RUN_ID
- <RUN_ID> or <COMMIT_SHA> of the image. Images in GitHub registry are
stored with those
+
+
+
+ -s, --github-image-id COMMIT_SHA
+ <COMMIT_SHA> of the image. Images in GitHub registry are stored with
those
to be able to easily find the image for particular CI runs. Once you
know the
- <RUN_ID> or <COMMIT_SHA>, you can specify it in github-image-id flag
and Breeze will
+ <COMMIT_SHA>, you can specify it in github-image-id flag and Breeze
will
automatically pull and use that image so that you can easily
reproduce a problem
that occurred in CI.
If you use this flag, automatically --use-github-registry is enabled.
-
Default: latest.
****************************************************************************************************
diff --git a/CI.rst b/CI.rst
index e7ed6af..8513652 100644
--- a/CI.rst
+++ b/CI.rst
@@ -60,7 +60,7 @@ and pass it to the "CI Build" workflow.
Currently in master version of Airflow we run tests in 3 different versions of
Python (3.6, 3.7, 3.8)
which means that we have to build 6 images (3 CI ones and 3 PROD ones). Yet we
run around 12 jobs
with each of the CI images. That is a lot of time to just build the
environment to run. Therefore
-we are utilising ``workflow_run`` feature of GitHub Actions.
+we are utilising ``pull_request_target`` feature of GitHub Actions.
This feature allows to run a separate, independent workflow, when the main
workflow is run -
this separate workflow is different than the main one, because by default it
runs using ``master`` version
@@ -70,11 +70,11 @@ This is especially important in our case where Pull
Requests to Airflow might co
and it would be a huge security issue if anyone from outside could
utilise the WRITE access to Apache Airflow repository via an external Pull
Request.
-Thanks to the WRITE access and fact that the 'workflow_run' by default uses
the 'master' version of the
+Thanks to the WRITE access and fact that the 'pull_request_target' by default
uses the 'master' version of the
sources, we can safely run some logic there will checkout the incoming Pull
Request, build the container
image from the sources from the incoming PR and push such image to an GitHub
Docker Registry - so that
this image can be built only once and used by all the jobs running tests. The
image is tagged with unique
-``RUN_ID`` of the incoming Pull Request and the tests run in the Pull Request
can simply pull such image
+``COMMIT_SHA`` of the incoming Pull Request and the tests run in the Pull
Request can simply pull such image
rather than build it from the scratch. Pulling such image takes ~ 1 minute,
thanks to that we are saving
a lot of precious time for jobs.
@@ -138,7 +138,7 @@ You can read more about Breeze in `BREEZE.rst
<BREEZE.rst>`_ but in essence it i
you to re-create CI environment in your local development instance and
interact with it. In its basic
form, when you do development you can run all the same tests that will be run
in CI - but locally,
before you submit them as PR. Another use case where Breeze is useful is when
tests fail on CI. You can
-take the ``RUN_ID`` of failed build pass it as ``--github-image-id`` parameter
of Breeze and it will
+take the full ``COMMIT_SHA`` of the failed build pass it as
``--github-image-id`` parameter of Breeze and it will
download the very same version of image that was used in CI and run it
locally. This way, you can very
easily reproduce any failed test that happens in CI - even if you do not check
out the sources
connected with the run.
@@ -467,10 +467,10 @@ the model of permission management is not the same for
Container Registry as it
| | | useful if
commit SHA is used as pull tag |
+--------------------------------+---------------------------+----------------------------------------------+
| GITHUB_REGISTRY_PULL_IMAGE_TAG | ``latest`` | Pull this image
tag. This is "latest" by |
-| | | default, can be
commit SHA or RUN_ID. |
+| | | default, can
also be full-length commit SHA. |
+--------------------------------+---------------------------+----------------------------------------------+
| GITHUB_REGISTRY_PUSH_IMAGE_TAG | ``latest`` | Pull this image
tag. This is "latest" by |
-| | | default, can be
commit SHA or RUN_ID. |
+| | | default, can
also be full-length commit SHA. |
+--------------------------------+---------------------------+----------------------------------------------+
Authentication in GitHub Registry
@@ -622,41 +622,39 @@ to file JIRA ticket to Apache Infra in order to get an
access).
Workflows
=========
+A general note about cancelling duplicated workflows: for the Build Images, CI
Build and CodeQL workflows we
+use the ``concurrency`` feature of GitHub actions to automatically cancel
"old" workflows of each time --
+meaning if you push a new commit to a branch or to a pull request and there is
a workflow running GitHub
+Actions will cancel the old job automatically.
+
Build Images Workflow
---------------------
-This workflow has two purposes - it builds images for the CI Workflow but also
it cancels duplicate or
-failed builds in order to save job time in GitHub Actions and allow for faster
feedback for developers.
+This workflow builds images for the CI Workflow.
-It's a special type of workflow: ``workflow_run`` which means that it is
triggered by other workflows (in our
-case it is triggered by the ``CI Build`` workflow). This also means that the
workflow has Write permission to
+It's a special type of workflow: ``pull_request_target`` which means that it
is triggered when a pull request is opened.
+This also means that the workflow has Write permission to
the Airflow repository and it can - for example - push to the GitHub registry
the images used by CI Builds
which means that the images can be built only once and reused by all the CI
jobs (including the matrix jobs).
We've implemented it in the way that the CI Build running will wait until the
images are built by the
"Build Images" workflow.
+(This workflow is also triggered on normal pushes to our "main" branches, i.e.
after a pull request is merged.)
+
It's possible to disable this feature and go back to the previous behaviour via
``GITHUB_REGISTRY_WAIT_FOR_IMAGE`` flag in the "Build Workflow image". Setting
it to "false" switches back to
the behaviour that each job builds own image.
You can also switch back to jobs building the images on its own on the fork
level by setting
-``AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE`` secret to ``false``. This will
disable pushing the "RUN_ID"
+``AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE`` secret to ``false``. This will
disable pushing the "COMMIT_SHA"
images to GitHub Registry and all the images will be built locally by each
job. It is about 20%
slower for the whole build on average, but it does not require to have access
to push images to
GitHub, which sometimes might be not available (depending on the account
status).
-The write permission also allows to cancel duplicate workflows. It is not
possible for the Pull Request
-CI Builds run from the forks as they have no Write permission allowing them to
cancels running workflows.
+The write permission also allows to cancel workflows. It is not possible for
the pull request
+"CI Builds" workflow run from the forks as they have no Write permission and
cannot cancel running workflows.
In our case we perform several different cancellations:
-* we cancel duplicate "CI Build" workflow runs s (i.e. workflows from the same
repository and branch that
- were started in quick succession - this allows to save workers that would
have been busy running older
- version of the same Pull Request (usually with fix-ups) and free them for
other runs.
-
-* we cancel duplicate "Build Images" workflow runs for the same reasons. The
"Build Images" builds run image
- builds which takes quite some time, so pushing a fixup quickly on the same
branch will also cancel the
- past "Build Images" workflows.
-
* last, but not least - we cancel any of the "CI Build" workflow runs that
failed in some important jobs.
This is another optimisations - GitHub does not have "fail-fast" on the
whole run and this cancelling
effectively implements "fail-fast" of runs for some important jobs. Note
that it only works when you
@@ -669,8 +667,6 @@ The workflow has the following jobs:
| Job | Description |
| | |
+===========================+=============================================+
-| Cancel workflow runs | Cancels duplicated and failed workflows |
-+---------------------------+---------------------------------------------+
| Build Info | Prints detailed information about the build |
+---------------------------+---------------------------------------------+
| Build CI/PROD images | Builds all configured CI and PROD images |
@@ -799,51 +795,47 @@ The image names follow the patterns:
+--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+
| Image | Name pattern | Tag for format |
Comment
|
+==============+============================+================================+============================================================================================+
-| Python image | Python | <X.Y>-slim-buster-<RUN_ID> |
Base Python image used by both production and CI image.
|
-| | | <X.Y>-slim-buster-<COMMIT_SHA> |
Python maintainer release new versions of those image with security fixes every
few weeks. |
+| Python image | Python | <X.Y>-slim-buster-<COMMIT_SHA> |
Base Python image used by both production and CI image.
|
+| | | |
Python maintainer release new versions of those image with security fixes every
few weeks. |
+--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+
-| CI image | <BRANCH>-python<X.Y>-ci | <RUN_ID> |
CI image - this is the image used for most of the tests.
|
-| | | <COMMIT_SHA> |
|
+| CI image | <BRANCH>-python<X.Y>-ci | <COMMIT_SHA> |
CI image - this is the image used for most of the tests.
|
+--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+
-| PROD Build | <BRANCH>-python<X.Y>-build | <RUN_ID> |
Production Build image - this is the "build" segment of production image.
|
-| image | | <COMMIT_SHA> |
It contains build-essentials and all necessary packages to install PIP
packages. |
+| PROD Build | <BRANCH>-python<X.Y>-build | <COMMIT_SHA> |
Production Build image - this is the "build" segment of production image.
|
+| image | | |
It contains build-essentials and all necessary packages to install PIP
packages. |
+--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+
-| PROD image | <BRANCH>-python<X.Y> | <RUN_ID> |
Production image. This is the actual production image - optimized for size.
|
-| | | <COMMIT_SHA> |
It contains only compiled libraries and minimal set of dependencies to run
Airflow. |
+| PROD image | <BRANCH>-python<X.Y> | <COMMIT_SHA> |
Production image. This is the actual production image - optimized for size.
|
+| | | |
It contains only compiled libraries and minimal set of dependencies to run
Airflow. |
+--------------+----------------------------+--------------------------------+--------------------------------------------------------------------------------------------+
* <BRANCH> might be either "master" or "v1-10-test" or "v2-*-test"
* <X.Y> - Python version (Major + Minor). For "master" and "v2-*-test" should
be in ["3.6", "3.7", "3.8"]. For
v1-10-test it should be in ["2.7", "3.5", "3.6". "3.7", "3.8"].
-* <RUN_ID> - GitHub Actions RUN_ID. You can get it from CI action job outputs
(run id is printed in
- logs and displayed as part of the step name. All PRs belong to some RUN_ID
and this way you can
- pull the very exact version of image used in that RUN_ID
-* <COMMIT_SHA> - for images that get merged to "master", "v2-*-test" of
"v1-10-test" the images are also tagged
- with the commit SHA of that particular commit. This way you can easily find
the image that was used
- for testing for that "master", "v2-*-test" or "v1-10-test" test run.
+* <COMMIT_SHA> - for images that get merged to "master", "v2-*-test" of
"v1-10-test", or built as part of a
+ pull request the images are tagged with the (full lenght) commit SHA of that
particular branch. For pull
+ requests the SHA used is the tip of the pull request branch.
Reproducing CI Runs locally
===========================
Since we store images from every CI run, you should be able easily reproduce
any of the CI build problems
locally. You can do it by pulling and using the right image and running it
with the right docker command,
-For example knowing that the CI build had 210056909 RUN_ID (you can find it
from GitHub CI logs):
+For example knowing that the CI build was for commit
``cd27124534b46c9688a1d89e75fcd137ab5137e3``:
.. code-block:: bash
- docker pull
docker.pkg.github.com/apache/airflow/master-python3.6-ci:210056909
+ docker pull
docker.pkg.github.com/apache/airflow/master-python3.6-ci:cd27124534b46c9688a1d89e75fcd137ab5137e3
- docker run -it
docker.pkg.github.com/apache/airflow/master-python3.6-ci:210056909
+ docker run -it
docker.pkg.github.com/apache/airflow/master-python3.6-ci:cd27124534b46c9688a1d89e75fcd137ab5137e3
But you usually need to pass more variables and complex setup if you want to
connect to a database or
enable some integrations. Therefore it is easiest to use `Breeze
<BREEZE.rst>`_ for that. For example if
-you need to reproduce a MySQL environment with kerberos integration enabled
for run 210056909, in python
-3.8 environment you can run:
+you need to reproduce a MySQL environment with kerberos integration enabled
for commit
+cd27124534b46c9688a1d89e75fcd137ab5137e3, in python 3.8 environment you can
run:
.. code-block:: bash
- ./breeze --github-image-id 210056909 --github-registry docker.pkg.github.com
--python 3.8
+ ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3
--github-registry docker.pkg.github.com --python 3.8
You will be dropped into a shell with the exact version that was used during
the CI run and you will
be able to run pytest tests manually, easily reproducing the environment that
was used in CI. Note that in
diff --git a/IMAGES.rst b/IMAGES.rst
index 51b2934..6f32e93 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -316,14 +316,14 @@ The images are linked to the repository via
``org.opencontainers.image.source``
Naming convention for GitHub Packages
-------------------------------------
-Images built as "Run ID snapshot":
+Images with a commit SHA (built for pull requests and pushes)
.. code-block:: bash
- docker.pkg.github.com.io/apache-airflow/<BRANCH>-pythonX.Y-ci-v2:<RUN_ID>
- for CI images
- docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-v2:<RUN_ID> -
for production images
- docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-build-v2:<RUN_ID> -
for production build stage
- docker.pkg.github.com/apache-airflow/python-v2:X.Y-slim-buster-<RUN_ID> -
for base Python images
+
docker.pkg.github.com.io/apache-airflow/<BRANCH>-pythonX.Y-ci-v2:<COMMIT_SHA> -
for CI images
+ docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-v2:<COMMIT_SHA>
- for production images
+
docker.pkg.github.com/apache-airflow/<BRANCH>-pythonX.Y-build-v2:<COMMIT_SHA> -
for production build stage
+ docker.pkg.github.com/apache-airflow/python-v2:X.Y-slim-buster-<COMMIT_SHA>
- for base Python images
Latest images (pushed when master merge succeeds):
@@ -332,20 +332,20 @@ Latest images (pushed when master merge succeeds):
docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-ci-v2:latest -
for CI images
docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-v2:latest -
for production images
docker.pkg.github.com/apache/airflow/<BRANCH>-pythonX.Y-build-v2:latest -
for production build stage
- docker.pkg.github.com/apache/airflow/python-v2:X.Y-slim-buster - for base
Python images
+ docker.pkg.github.com/apache/airflow/python-v2:X.Y-slim-buster -
for base Python images
Naming convention for GitHub Container Registry
-----------------------------------------------
-Images built as "Run ID snapshot":
+Images with a commit SHA (built for pull requests and pushes)
.. code-block:: bash
- ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-ci-v2:<RUN_ID> -
for CI images
- ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-v2:<RUN_ID> -
for production images
- ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-build-v2:<RUN_ID> -
for production build stage
- ghcr.io/apache/airflow-python-v2:X.Y-slim-buster-<RUN_ID> - for base Python
images
+ ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-ci-v2:<COMMIT_SHA> - for CI
images
+ ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-v2:<COMMIT_SHA> - for
production images
+ ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-build-v2:<COMMIT_SHA> - for
production build stage
+ ghcr.io/apache/airflow-python-v2:X.Y-slim-buster-<COMMIT_SHA> - for base
Python images
Latest images (pushed when master merge succeeds):
@@ -354,7 +354,7 @@ Latest images (pushed when master merge succeeds):
ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-ci-v2:latest - for CI images
ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-v2:latest - for production
images
ghcr.io/apache/airflow-<BRANCH>-pythonX.Y-build-v2:latest - for production
build stage
- ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python images
+ ghcr.io/apache/airflow-python-v2:X.Y-slim-buster - for base Python
images
Note that we never push or pull "release" images to GitHub registry. It is
only used for CI builds
diff --git a/breeze b/breeze
index 036c7ae..50f9e55 100755
--- a/breeze
+++ b/breeze
@@ -1164,7 +1164,7 @@ function breeze::parse_arguments() {
echo "GitHub image id: ${2}"
echo
echo "Force pulling the image, using GitHub registry and skip
mounting local sources."
- echo "This is in order to get the exact same version as used in CI
environment for SHA/RUN_ID!."
+ echo "This is in order to get the exact same version as used in CI
environment for SHA!."
echo "You can specify --skip-mounting-local-sources to not mount
local sources to get exact. "
echo "behaviour as in the CI environment."
echo
@@ -1716,20 +1716,15 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
'${CMDNAME} shell -- -c \"ls -la\"'
'${CMDNAME} -- -c \"ls -la\"'
- For DockerHub pull --dockerhub-user and --dockerhub-repo flags can be
used to specify
+ For DockerHub pull: --dockerhub-user and --dockerhub-repo flags can be
used to specify
the repository to pull from. For GitHub repository, the
--github-repository
- flag can be used for the same purpose. You can also use
- --github-image-id <COMMIT_SHA>|<RUN_ID> in case you want to pull the
image
- with specific COMMIT_SHA tag or RUN_ID.
+ flag can be used for the same purpose. You can also use
--github-image-id <COMMIT_SHA> in case
+ you want to pull the image with specific COMMIT_SHA tag.
'${CMDNAME} shell \\
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' -
pull/use image with SHA
'${CMDNAME} \\
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' -
pull/use image with SHA
- '${CMDNAME} shell \\
- --github-image-id 209845560' - pull/use image with RUN_ID
- '${CMDNAME} \\
- --github-image-id 209845560' - pull/use image with RUN_ID
Most flags are applicable to the shell command as it will run build when
needed.
"
@@ -1772,18 +1767,15 @@ ${CMDNAME} build-image [FLAGS]
You can also pass '--production-image' flag to build production image
rather than CI image.
- For DockerHub pull. '--dockerhub-user' and '--dockerhub-repo' flags can
be used to specify
- the repository to pull from. For GitHub repository, the
'--github-repository'
- flag can be used for the same purpose. You can also use
- '--github-image-id <COMMIT_SHA>|<RUN_ID>' in case you want to pull the
image with
- specific COMMIT_SHA tag or RUN_ID.
-
+ For DockerHub pulling of base images: '--dockerhub-user' and
'--dockerhub-repo' flags can be
+ used to specify the repository to pull from. For GitHub repository, the
'--github-repository'
+ flag can be used for the same purpose.
Flags:
$(breeze::flag_airflow_variants)
$(breeze::flag_build_different_airflow_version)
$(breeze::flag_production_image)
$(breeze::flag_build_docker_images)
-$(breeze::flag_pull_push_docker_images)
+$(breeze::flag_pull_push_docker_images "no_show_sha")
$(breeze::flag_verbosity)
"
readonly DETAILED_USAGE_BUILD_IMAGE
@@ -1946,11 +1938,11 @@ ${CMDNAME} push_image [FLAGS]
Pushes images to docker registry. You can push the images to DockerHub
registry (default)
or to the GitHub registry (if --use-github-registry flag is used).
- For DockerHub pushes --dockerhub-user and --dockerhub-repo flags can be
used to specify
+ For DockerHub pushes: --dockerhub-user and --dockerhub-repo flags can be
used to specify
the repository to push to. For GitHub repository, the --github-repository
flag can be used for the same purpose. You can also add
- --github-image-id <COMMIT_SHA>|<RUN_ID> in case you want to push image
with specific
- SHA tag or run id. In case you specify --github-repository or
--github-image-id, you
+ --github-image-id <COMMIT_SHA> in case you want to push image with
specific
+ SHA tag. In case you specify --github-repository or --github-image-id,
you
do not need to specify --use-github-registry flag.
You can also add --production-image flag to switch to production image
(default is CI one)
@@ -1965,8 +1957,6 @@ ${CMDNAME} push_image [FLAGS]
--github-repository user/airflow' - to push to your user's fork
'${CMDNAME} push-image \\
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - to
push with COMMIT_SHA
- '${CMDNAME} push-image \\
- --github-image-id 209845560' - to push with RUN_ID
Flags:
$(breeze::flag_pull_push_docker_images)
@@ -2785,6 +2775,7 @@ Build options:
# Flag information.
#######################################################################################################
function breeze::flag_pull_push_docker_images() {
+ local show_sha="${1-'show_sha'}"
echo "
-D, --dockerhub-user DOCKERHUB_USER
DockerHub user used to pull, push and build images. Default:
${_breeze_default_dockerhub_user:=}.
@@ -2813,16 +2804,19 @@ ${FORMATTED_GITHUB_REGISTRY}
If you use this flag, automatically --use-github-registry flag is
enabled.
--s, --github-image-id COMMIT_SHA|RUN_ID
- <RUN_ID> or <COMMIT_SHA> of the image. Images in GitHub registry are
stored with those
+"
+
+ [[ "$show_sha" != "no_show_sha" ]] && echo "
+
+-s, --github-image-id COMMIT_SHA
+ <COMMIT_SHA> of the image. Images in GitHub registry are stored with
those
to be able to easily find the image for particular CI runs. Once you
know the
- <RUN_ID> or <COMMIT_SHA>, you can specify it in github-image-id flag
and Breeze will
+ <COMMIT_SHA>, you can specify it in github-image-id flag and Breeze
will
automatically pull and use that image so that you can easily reproduce
a problem
that occurred in CI.
If you use this flag, automatically --use-github-registry is enabled.
-
Default: ${_breeze_default_github_image_id:=}.
"
diff --git a/images/ci/pull_request_ci_flow.md5
b/images/ci/pull_request_ci_flow.md5
index 0da67b0..d1ff43a 100644
--- a/images/ci/pull_request_ci_flow.md5
+++ b/images/ci/pull_request_ci_flow.md5
@@ -1 +1 @@
-ad4af91ca126141b54e67c8741fa50ab images/ci/pull_request_ci_flow.mermaid
+7a3a1def02205e15a6c91fb2a5605edf images/ci/pull_request_ci_flow.mermaid
diff --git a/images/ci/pull_request_ci_flow.mermaid
b/images/ci/pull_request_ci_flow.mermaid
index c4729d3..f97e1cd 100644
--- a/images/ci/pull_request_ci_flow.mermaid
+++ b/images/ci/pull_request_ci_flow.mermaid
@@ -16,37 +16,23 @@
%% under the License.
sequenceDiagram
- Contributor -->> Fork: Submit PR
Fork -->> Airflow Repo: Start Pull Request
- activate Airflow Repo
- Note over Airflow Repo: Trigger PR build
Note over CI Build: pull_request<br>[Read Token]
- Note over Build Images: workflow_run<br>[Write Token]
+ Note over Build Images: pull_request_target<br>[Write Token]
+ activate Airflow Repo
Airflow Repo -->> CI Build: Trigger 'pull_request' event
activate CI Build
- CI Build -->> Build Images: Trigger 'workflow_run' event
activate Build Images
- Note over Build Images: Find Duplicate<br>[CI Build]
- opt Duplicated builds
- Build Images -->> CI Build: Cancel
- end
- Note over Build Images: Find Duplicates<br>[Build Image]
- opt Duplicated builds
- Build Images -->> Build Images: Cancel
- end
- Note over Build Images: Find failed<br>[CI Build]
+ Airflow Repo -->> Build Images: Trigger 'pull_request_target' event
opt Failed builds
+ Note right of Build Images: Find failed<br>[CI Build]
Build Images -->> CI Build: Cancel
end
- Note over Build Images: Find failed<br>[Build Image]
- opt Failed builds
- Build Images -->> Build Images: Cancel
- end
Note over Build Images: Build info
par 3.6, 3.7, 3.8
- Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID]
+ Note over Build Images: Build PROD Images<br>[COMMIT_SHA]
and
- Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID]
+ Note over Build Images: Build CI Images<br>[COMMIT_SHA]
end
par No CI image
Note over CI Build: Build info
@@ -64,17 +50,17 @@ sequenceDiagram
Build Images ->> GitHub Registry: Push PROD Images
end
opt If any step failed
- Build Images -->> CI Build: Cancel triggering run
+ Build Images -->> CI Build: Cancel associated run
end
deactivate Build Images
- Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID]
+ Note over GitHub Registry: Tagged Images<br>[COMMIT_SHA]
loop Wait for images
par 3.6, 3.7, 3.8
CI Build ->> CI Build: Pull CI Images
- Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+ Note over CI Build: Wait for<br>[COMMIT_SHA]
and
CI Build ->> CI Build: Pull PROD Images
- Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+ Note over CI Build: Wait for<br>[COMMIT_SHA]
end
end
par 3.6, 3.7, 3.8
diff --git a/images/ci/pull_request_ci_flow.png
b/images/ci/pull_request_ci_flow.png
index 04aa6b5..12f75d9 100644
Binary files a/images/ci/pull_request_ci_flow.png and
b/images/ci/pull_request_ci_flow.png differ
diff --git a/images/ci/push_ci_flow.md5 b/images/ci/push_ci_flow.md5
index 021419a..40f0cb0 100644
--- a/images/ci/push_ci_flow.md5
+++ b/images/ci/push_ci_flow.md5
@@ -1 +1 @@
-2e735f22a91bec2ce043289dc2f7f824 images/ci/push_ci_flow.mermaid
+b879f9a8009677fc5248d849e15e4a4a images/ci/push_ci_flow.mermaid
diff --git a/images/ci/push_ci_flow.mermaid b/images/ci/push_ci_flow.mermaid
index a7bd2d3..afb5389 100644
--- a/images/ci/push_ci_flow.mermaid
+++ b/images/ci/push_ci_flow.mermaid
@@ -18,36 +18,21 @@
sequenceDiagram
Committer -->> Airflow Repo: Merge PR
activate Airflow Repo
- Note over Airflow Repo: Trigger PR build
- Note over CI Build: push<br>[Write Token]
- Note over Build Images: workflow_run<br>[Write Token]
- Airflow Repo -->> CI Build: Trigger 'pull_request' event
+ Airflow Repo -->> CI Build: Trigger 'schedled' event
activate CI Build
- CI Build -->> Build Images: Trigger 'workflow_run' event
activate Build Images
- Note over Build Images: Find Duplicate<br>[CI Build]
- opt Duplicated builds
- Build Images -->> CI Build: Cancel
- end
- Note over Build Images: Find Duplicates<br>[Build Image]
- opt Duplicated builds
- Build Images -->> Build Images: Cancel
- end
- Note over Build Images: Find failed<br>[CI Build]
+ Airflow Repo -->> Build Images: Trigger 'schedled' event
opt Failed builds
+ Note right of Build Images: Find failed<br>[CI Build]
Build Images -->> CI Build: Cancel
end
- Note over Build Images: Find failed<br>[Build Image]
- opt Failed builds
- Build Images -->> Build Images: Cancel
- end
Note over Build Images: Build info
par 3.6, 3.7, 3.8
Note over Build Images: Update constraints
- Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID]
+ Note over Build Images: Build PROD Images<br>[COMMIT_SHA]
and
Note over Build Images: Update constraints
- Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID]
+ Note over Build Images: Build CI Images<br>[COMMIT_SHA]
end
par No CI image
Note over CI Build: Build info
@@ -65,17 +50,17 @@ sequenceDiagram
Build Images ->> GitHub Registry: Push PROD Images
end
opt If any step failed
- Build Images -->> CI Build: Cancel triggering run
+ Build Images -->> CI Build: Cancel associated run
end
deactivate Build Images
- Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID]
+ Note over GitHub Registry: Tagged Images<br>[COMMIT_SHA]
loop Wait for images
par 3.6, 3.7, 3.8
CI Build ->> CI Build: Pull CI Images
- Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+ Note over CI Build: Wait for<br>[COMMIT_SHA]
and
CI Build ->> CI Build: Pull PROD Images
- Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+ Note over CI Build: Wait for<br>[COMMIT_SHA]
end
end
par 3.6, 3.7, 3.8
diff --git a/images/ci/push_ci_flow.png b/images/ci/push_ci_flow.png
index 9f791f2..a388b23 100644
Binary files a/images/ci/push_ci_flow.png and b/images/ci/push_ci_flow.png
differ
diff --git a/scripts/ci/libraries/_build_images.sh
b/scripts/ci/libraries/_build_images.sh
index 775a6bf..63a8e08 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -386,39 +386,6 @@ function build_images::get_docker_image_names() {
# File that is touched when the CI image is built for the first time
locally
export
BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
- # This is 1-1 mapping of image names of Apache Airflow stored in DockerHub
vs. the same images stored
- # in GitHub Registries (either GitHub Container Registry or GitHub
Packages)
- #
- # We have to apply naming conventions used by the registries and keep
multiple RUN_ID tags. We use
- # common suffix ('gcr-v1') to be able to switch to different set of cache
images if needed
- # - for example when some images gets broken (might happen with GitHub
Actions Registries) or when
- # the storage capacity per image is reached (though it is apparently
unlimited)
- #
- # Some examples:
- #
- # In case of GitHub Container Registry:
- #
- # * Prod Image: "apache/airflow:master-python3.8" ->
"apache/airflow-master-python3.8-gcr-v1:<RUN_ID>"
- # * Prod build image: "apache/airflow:master-python3.8-build" ->
"apache/airflow-master-python3.8-build-gcr-v1:<RUN_ID>"
- # * CI build image: "apache/airflow:master-python3.8-ci" ->
"apache/airflow-master-python3.8-ci-gcr-v1:<RUN_ID>"
- #
- # The python base image/tag mapping is slightly different (the base images
are shared by all Prod/Build/CI images)
- # And python version is part of the tag.
- #
- # "apache/airflow:python-3.6 ->
"apache/airflow-python-gcr-v1:3.6-slim-buster-<RUN_ID>"
- #
- # In case of GitHub Packages image must be part of the repository:
- #
- # * Prod Image: "apache/airflow:master-python3.8" ->
"apache/airflow/master-python3.8-gcr-v1:<RUN_ID>"
- # * Prod build image: "apache/airflow:master-python3.8-build" ->
"apache/airflow/master-python3.8-build-gcr-v1:<RUN_ID>"
- # * CI build image: "apache/airflow:master-python3.8-ci" ->
"apache/airflow/master-python3.8-ci-gcr-v1:<RUN_ID>"
- #
- # The python base image/tag mapping is slightly different (the base images
are shared by all
- # Prod/Build/CI images) and python version is part of the tag.
- #
- # "apache/airflow:python-3.6 ->
"apache/airflow/python/gcr-v1:3.6-slim-buster-<RUN_ID>"
-
-
local image_name
image_name="${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)"
local image_separator
@@ -433,10 +400,24 @@ function build_images::get_docker_image_names() {
exit 1
fi
+ # Example:
+ # docker.pkg.github.com/apache/airflow/master-python3.6-v2
+ # ghcr.io/apache/airflow-v2-1-test-python-v2:3.6-slim-buster
+ # ghcr.io/apache/airflow-python-v2:3.6-slim-buster-<COMMIT_SHA>
export
GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${image_name}${image_separator}${AIRFLOW_PROD_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
+ # Example:
+ # docker.pkg.github.com/apache/airflow/master-python3.6-build-v2
+ # ghcr.io/apache/airflow-master-python3.6-build-v2
export
GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${image_name}${image_separator}${AIRFLOW_PROD_BASE_TAG}-build${GITHUB_REGISTRY_IMAGE_SUFFIX}"
+
+ # Example:
+ # docker.pkg.github.com/apache/airflow/python-v2:3.6-slim-buster
+ # ghcr.io/apache/airflow-python-v2:3.6-slim-buster
+ # ghcr.io/apache/airflow-python-v2:3.6-slim-buster-<COMMIT_SHA>
export
GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${image_name}${image_separator}python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+ # Example:
+ # docker.pkg.github.com/apache/airflow/master-python3.8-ci-v2
export
GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${image_name}${image_separator}${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
}
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh
b/scripts/ci/libraries/_push_pull_remove_images.sh
index 932797c..d7bc17a 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -204,11 +204,11 @@ function
push_pull_remove_images::push_ci_images_to_dockerhub() {
# Push image to GitHub registry with the push tag:
-# "${GITHUB_RUN_ID}" - in case of pull-request triggered 'workflow_run'
builds
-# "latest" - in case of push builds
+# "${COMMIT_SHA}" - in case of pull-request triggered 'workflow_run' builds
+# "latest" - in case of push builds
# Push python image to GitHub registry with the push tag:
-# X.Y-slim-buster-"${GITHUB_RUN_ID}" - in case of pull-request triggered
'workflow_run' builds
-# X.Y-slim-buster - in case of push builds
+# X.Y-slim-buster-"${COMMIT_SHA}" - in case of pull-request triggered
'workflow_run' builds
+# X.Y-slim-buster - in case of push builds
function push_pull_remove_images::push_python_image_to_github() {
local python_tag_suffix=""
if [[ ${GITHUB_REGISTRY_PUSH_IMAGE_TAG} != "latest" ]]; then
@@ -262,8 +262,8 @@ function
push_pull_remove_images::push_prod_images_to_dockerhub () {
# Pushes PROD image to and their tags to registry in GitHub
# Push image to GitHub registry with chosen push tag
# the PUSH tag might be:
-# "${GITHUB_RUN_ID}" - in case of pull-request triggered 'workflow_run'
builds
-# "latest" - in case of push builds
+# "${COMMIT_SHA}" - in case of pull-request triggered 'workflow_run' builds
+# "latest" - in case of push builds
function push_pull_remove_images::push_prod_images_to_github () {
local
airflow_prod_tagged_image="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_PROD_IMAGE}" "${airflow_prod_tagged_image}"
