This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-0-stable in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0b594d9245b73b6e40f769f497574f540c9e3c16 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Jan 12 15:42:28 2025 +0100 Remove pull_request_target workflow --- .github/workflows/build-images-workflow-run.yml | 562 --------- .github/workflows/ci.yml | 1228 -------------------- .github/workflows/codeql-analysis.yml | 99 -- .github/workflows/label_when_reviewed.yml | 29 - .../workflows/label_when_reviewed_workflow_run.yml | 172 --- .github/workflows/repo_sync.yml | 36 - .github/workflows/scheduled_quarantined.yml | 107 -- 7 files changed, 2233 deletions(-) diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml deleted file mode 100644 index e00cdc5fd0d..00000000000 --- a/.github/workflows/build-images-workflow-run.yml +++ /dev/null @@ -1,562 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: "Build Images" -on: # yamllint disable-line rule:truthy - workflow_run: - workflows: ["CI Build"] - types: ['requested'] -env: - MOUNT_SELECTED_LOCAL_SOURCES: "false" - FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "false" - CHECK_IMAGE_FOR_REBUILD: "true" - SKIP_CHECK_REMOTE_IMAGE: "true" - DB_RESET: "true" - VERBOSE: "true" - USE_GITHUB_REGISTRY: "true" - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} - # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the - # Airflow one is going to be used - CONSTRAINTS_GITHUB_REPOSITORY: >- - ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }} - # This token is WRITE one - workflow_run type of events always have the WRITE token - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # This token should not be empty in workflow_run type of event. - CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }} - GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" - GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" - BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} - INSTALL_PROVIDERS_FROM_SOURCES: "true" - GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }} - -jobs: - - cancel-workflow-runs: - timeout-minutes: 10 - name: "Cancel workflow runs" - runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} - outputs: - sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - sourceHeadBranch: ${{ steps.source-run-info.outputs.sourceHeadBranch }} - sourceHeadSha: ${{ steps.source-run-info.outputs.sourceHeadSha }} - mergeCommitSha: ${{ steps.source-run-info.outputs.mergeCommitSha }} - targetCommitSha: ${{ steps.source-run-info.outputs.targetCommitSha }} - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} - targetBranch: ${{ steps.source-run-info.outputs.targetBranch }} - sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }} - cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }} - buildImages: ${{ steps.build-images.outputs.buildImages }} - runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Get information about the original trigger of the run" - uses: ./.github/actions/get-workflow-origin - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - sourceRunId: ${{ github.event.workflow_run.id }} - - name: "Cancel duplicated 'CI Build' runs" - uses: ./.github/actions/cancel-workflow-runs - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - sourceRunId: ${{ github.event.workflow_run.id }} - - name: "Output BUILD_IMAGES" - id: build-images - run: | - # Workaround - jobs cannot access env variable in "ifs" - # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16 - echo "::set-output name=buildImages::${BUILD_IMAGES}" - - name: "Cancel duplicated 'Build Image' runs" - # We find duplicates of our own "Build Image" runs - due to a missing feature - # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching - # trick ¯\_(ツ)_/¯. We name the build-info job appropriately - # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the - # current Event/Repo/Branch combination. - uses: ./.github/actions/cancel-workflow-runs - with: - cancelMode: namedJobs - token: ${{ secrets.GITHUB_TOKEN }} - notifyPRCancel: true - jobNameRegexps: > - [".*Event: ${{ steps.source-run-info.outputs.sourceEvent }} - Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }}.*"] - if: env.BUILD_IMAGES == 'true' - - name: "Cancel all 'CI Build' runs where some jobs failed" - # We find any of the "CI Build" workflow runs, where any of the important jobs - # failed. The important jobs are selected by the regexp array below. - # We also produce list of canceled "CI Build' runs as output, so that we - # can cancel all the matching "Build Images" workflow runs in the two following steps. - # Yeah. Adding to the complexity ¯\_(ツ)_/¯. - uses: ./.github/actions/cancel-workflow-runs - id: cancel-failed - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: failedJobs - sourceRunId: ${{ github.event.workflow_run.id }} - notifyPRCancel: true - jobNameRegexps: > - ["^Pylint$", "^Static checks", "^Build docs$", "^Spell check docs$", - "^Provider packages", "^Checks: Helm tests$", "^Test OpenAPI*"] - - name: "Extract canceled failed runs" - # We use this step to build regexp that will be used to match the Source Run id in - # the build-info job below. If we cancelled some "CI Build" runs in the "cancel-failed' step - # above - we want to cancel also the corresponding "Build Images" runs. Again we have - # to match the jobs using job name rather than use proper API because that feature - # is currently missing in GitHub Actions ¯\_(ツ)_/¯. - id: extract-cancelled-failed-runs - if: steps.cancel-failed.outputs.cancelledRuns != '[]' - run: | - REGEXP="Source Run id: " - SEPARATOR="" - for run_id in $(echo "${{ steps.cancel-failed.outputs.cancelledRuns }}" | jq '.[]') - do - REGEXP="${REGEXP}${SEPARATOR}(${run_id})" - SEPARATOR="|" - done - echo "::set-output name=matching-regexp::[\"${REGEXP}\"]" - - name: "Cancel triggered 'Build Images' runs for the cancelled failed runs" - # In case we do have some cancelled jobs in the "cancel-failed" step above - # We take the extracted regexp array prepared in the previous step and we use - # it to cancel any jobs that have matching names containing Source Run Id: - # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯. - if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]' - uses: ./.github/actions/cancel-workflow-runs - with: - cancelMode: namedJobs - token: ${{ secrets.GITHUB_TOKEN }} - notifyPRCancel: true - jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }} - - name: "Cancel duplicated 'CodeQL' runs" - uses: ./.github/actions/cancel-workflow-runs - id: cancel - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: allDuplicates - workflowFileName: 'codeql-analysis.yml' - - name: "Set Docker Cache Directive" - id: cache-directive - run: | - if [[ ${{ steps.source-run-info.outputs.sourceEvent }} == 'schedule' ]]; then - echo "::set-output name=docker-cache::disabled" - else - echo "::set-output name=docker-cache::pulled" - fi - - name: "Cancel all duplicated 'Build Image' runs" - # We find duplicates of all "Build Image" runs - due to a missing feature - # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching - # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match - # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those - # This might cancel own run, so this is the last step in the job - uses: ./.github/actions/cancel-workflow-runs - with: - cancelMode: allDuplicatedNamedJobs - token: ${{ secrets.GITHUB_TOKEN }} - notifyPRCancel: true - selfPreservation: false - jobNameRegexps: '["Event: \\S* Repo: \\S* Branch: \\S* "]' - - build-info: - # The name is such long because we are using it to cancel duplicated 'Build Images' runs - # by matching Event/Repo/Branch. This is a workaround for a missing feature of GitHub - # Actions to link the source workflow run and the triggered workflow_run one. - # We are also cancelling SourceRunId in case we determine that we should cancel the source - # Run because of some failing jobs in the source run. Again ¯\_(ツ)_/¯. - name: > - Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} - Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} - Run id: ${{ github.run_id }} - Source Run id: ${{ github.event.workflow_run.id }} - Sha: ${{ github.sha }} - Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} - Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }} - needs: [cancel-workflow-runs] - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - pythonVersions: ${{ steps.selective-checks.python-versions }} - upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - run-tests: ${{ steps.selective-checks.outputs.run-tests }} - run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - image-build: ${{ steps.selective-checks.outputs.image-build }} - if: > - needs.cancel-workflow-runs.outputs.buildImages == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: > - Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} - Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} - Run id: ${{ github.run_id }} - Source Run id: ${{ github.event.workflow_run.id }} - Sha: ${{ github.sha }} - Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} - Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - run: printenv - - name: > - Fetch incoming commit ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} with its parent - uses: actions/checkout@v2 - with: - ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - fetch-depth: 2 - persist-credentials: false - submodules: recursive - if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request' - # checkout the master version again, to use the right script in master workflow - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Selective checks - id: selective-checks - env: - EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - TARGET_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels }} - run: | - if [[ ${EVENT_NAME} == "pull_request" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi - - build-ci-images: - timeout-minutes: 80 - name: "Build CI images ${{matrix.python-version}}" - runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }} - needs: [build-info, cancel-workflow-runs] - strategy: - matrix: - # We need to attempt to build all possible versions here because workflow_run - # event is run from master for both master and v1-10-tests - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true - if: > - needs.build-info.outputs.image-build == 'true' && - needs.cancel-workflow-runs.outputs.buildImages == 'true' - env: - RUNS_ON: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn)[0] }} - BACKEND: postgres - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }} - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - CONTINUE_ON_PIP_CHECK_FAILURE: "true" - DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} - FORCE_PULL_BASE_PYTHON_IMAGE: > - ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' || 'false' }} - steps: - - name: > - Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}] - Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} - Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} - Run id: ${{ github.run_id }} - Source Run id: ${{ github.event.workflow_run.id }} - Sha: ${{ github.sha }} - Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} - Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - uses: actions/checkout@v2 - with: - ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - - name: "Retrieve DEFAULTS from the _initialization.sh" - # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands - # Also 2.7 and 3.5 versions are not allowed to proceed on master - id: defaults - run: | - DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV - DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ - scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV - if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \ - ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \ - ]]; then - echo "::set-output name=proceed::false" - else - echo "::set-output name=proceed::true" - fi - - name: > - Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder - to use ci/scripts from there. - uses: actions/checkout@v2 - with: - path: "main-airflow" - ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" - persist-credentials: false - submodules: recursive - if: steps.defaults.outputs.proceed == 'true' - - name: Initiate GitHub Checks for Building image - # Use the submodule from main, not the PR branch - uses: ./main-airflow/.github/actions/checks-action - id: build-image-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Status of image build CI: ${{ matrix.python-version }}" - status: "in_progress" - sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Building the image: CI: ${{ matrix.python-version }}. See the - [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - for details" } - if: steps.defaults.outputs.proceed == 'true' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - if: steps.defaults.outputs.proceed == 'true' - - name: > - Override "scripts/ci" with the "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch - so that the PR does not override it - # We should not override those scripts which become part of the image as they will not be - # changed in the image built - we should only override those that are executed to build - # the image. - run: | - rm -rf "scripts/ci" - mv "main-airflow/scripts/ci" "scripts" - if: steps.defaults.outputs.proceed == 'true' - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: steps.defaults.outputs.proceed == 'true' - - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - if: steps.defaults.outputs.proceed == 'true' - - name: "Push CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" - run: ./scripts/ci/images/ci_push_ci_images.sh - if: steps.defaults.outputs.proceed == 'true' - - name: Update GitHub Checks for Building image with status - uses: ./main-airflow/.github/actions/checks-action - if: always() && steps.defaults.outputs.proceed == 'true' - with: - token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.build-image-check.outputs.check_id }} - status: "completed" - sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - conclusion: ${{ job.status }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Building the image: CI: ${{ matrix.python-version }}. See the - [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - for details" } - - build-prod-images: - timeout-minutes: 80 - name: "Build PROD images ${{matrix.python-version}}" - runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }} - needs: [build-info, cancel-workflow-runs, build-ci-images] - strategy: - matrix: - # We need to attempt to build all possible versions here because workflow_run - # event is run from master for both master and v1-10-tests - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true - if: > - needs.build-info.outputs.image-build == 'true' && - needs.cancel-workflow-runs.outputs.buildImages == 'true' - env: - RUNS_ON: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn)[0] }} - BACKEND: postgres - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }} - GITHUB_REGISTRY_PULL_IMAGE_TAG: ${{ github.event.workflow_run.id }} - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} - FORCE_PULL_BASE_PYTHON_IMAGE: > - ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' || 'false' }} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" - steps: - - name: > - Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}] - Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - Repo: ${{ needs.cancel-workflow-runs.outputs.sourceHeadRepo }} - Branch: ${{ needs.cancel-workflow-runs.outputs.sourceHeadBranch }} - Run id: ${{ github.run_id }} - Source Run id: ${{ github.event.workflow_run.id }} - Sha: ${{ github.sha }} - Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }} - Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - uses: actions/checkout@v2 - with: - ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - - name: "Retrieve DEFAULTS from the _initialization.sh" - # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands - # Also 2.7 and 3.5 versions are not allowed to proceed on master - id: defaults - run: | - DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV - DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ - scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV - if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \ - ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \ - ]]; then - echo "::set-output name=proceed::false" - else - echo "::set-output name=proceed::true" - fi - - name: > - Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder - to use ci/scripts from there. - uses: actions/checkout@v2 - with: - path: "main-airflow" - ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" - persist-credentials: false - submodules: recursive - if: steps.defaults.outputs.proceed == 'true' - - name: Initiate GitHub Checks for Building image with status - uses: ./main-airflow/.github/actions/checks-action - id: build-image-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Status of image build PROD: ${{ matrix.python-version }}" - status: "in_progress" - sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Building the image: PROD: ${{ matrix.python-version }}. See the - [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - for details" } - if: steps.defaults.outputs.proceed == 'true' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - if: steps.defaults.outputs.proceed == 'true' - - name: > - Override "scripts/ci" with the "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch - so that the PR does not override it - # We should not override those scripts which become part of the image as they will not be - # changed in the image built - we should only override those that are executed to build - # the image. - run: | - rm -rf "scripts/ci" - mv "main-airflow/scripts/ci" "scripts" - if: steps.defaults.outputs.proceed == 'true' - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: steps.defaults.outputs.proceed == 'true' - - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - # Pull images built in the previous step - if: steps.defaults.outputs.proceed == 'true' - env: - GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" - - name: "Build PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - if: steps.defaults.outputs.proceed == 'true' - - name: "Push PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" - run: ./scripts/ci/images/ci_push_production_images.sh - if: steps.defaults.outputs.proceed == 'true' - - name: Update GitHub Checks for Building image with status - uses: ./main-airflow/.github/actions/checks-action - if: always() && steps.defaults.outputs.proceed == 'true' - with: - token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.build-image-check.outputs.check_id }} - status: "completed" - sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} - conclusion: ${{ job.status }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Building the image: PROD: ${{ matrix.python-version }}. See the - [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - for details" } - - cancel-on-build-cancel: - name: "Cancel 'CI Build' jobs on build image cancelling." - runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }} - if: cancelled() - needs: [cancel-workflow-runs, build-ci-images, build-prod-images] - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: ./.github/actions/cancel-workflow-runs - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: self - notifyPRCancel: true - notifyPRCancelMessage: "Building image for the PR has been cancelled" - sourceRunId: ${{ github.event.workflow_run.id }} - - cancel-on-build-failure: - name: "Cancel 'CI Build' jobs on build image failing." - runs-on: ${{ fromJson(needs.cancel-workflow-runs.outputs.runsOn) }} - if: failure() - needs: [cancel-workflow-runs, build-ci-images, build-prod-images] - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: ./.github/actions/cancel-workflow-runs - with: - token: ${{ secrets.GITHUB_TOKEN }} - cancelMode: self - notifyPRCancel: true - notifyPRCancelMessage: | - Building images for the PR has failed. Follow the workflow link to check the reason. - sourceRunId: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 6311b400016..00000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,1228 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: CI Build -on: # yamllint disable-line rule:truthy - schedule: - - cron: '28 0 * * *' - push: - branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test'] - pull_request: - branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test'] - -env: - MOUNT_SELECTED_LOCAL_SOURCES: "false" - FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "false" - CHECK_IMAGE_FOR_REBUILD: "true" - SKIP_CHECK_REMOTE_IMAGE: "true" - DB_RESET: "true" - VERBOSE: "true" - DOCKER_CACHE: "pulled" - USE_GITHUB_REGISTRY: "true" - # Might be either 'ghcr.io' or 'docker.pkg.github.com' - GITHUB_REGISTRY: "docker.pkg.github.com" - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} - # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the - # Airflow one is going to be used - CONSTRAINTS_GITHUB_REPOSITORY: >- - ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }} - # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # In builds from forks, this token is empty, and this is good because such builds do not even try - # to push images to the registry. - CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }} - GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}" - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - INSTALL_PROVIDERS_FROM_SOURCES: "true" - - # You can switch between building the image in "Build Images" workflow or building them in CI workflow - # Separately for each job. - # - # a) Using images build in the separate "Build Image" workflow: - # - # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" - # - # b) Building images in CI workflow - separately for each job: - # - # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" - # - # You can also switch back to building images locally and disabling the "Build Images" workflow - # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false" - GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} - -jobs: - - build-info: - name: "Build info" - # The runs-on cannot refer to env. or secrets. context, so we have no - # option but to specify a hard-coded list here. This is "safe", as the list - # is checked again by the runner using it's own list, so a PR author cannot - # change this and get access to our self-hosted runners - # - # When changing this list, ensure that it is kept in sync with the - # configOverride parameter in AWS SSM (which is what the runner uses) - runs-on: >- - ${{ ( - ( - github.event_name == 'push' || - github.event_name == 'schedule' || - contains(fromJSON('[ - "BasPH", - "Fokko", - "KevinYang21", - "XD-DENG", - "aijamalnk", - "alexvanboxel", - "aoen", - "artwr", - "ashb", - "bolkedebruin", - "criccomini", - "dimberman", - "feng-tao", - "houqp", - "jghoman", - "jmcarp", - "kaxil", - "leahecole", - "mik-laj", - "milton0825", - "mistercrunch", - "msumit", - "potiuk", - "r39132", - "ryanahamilton", - "ryw", - "saguziel", - "sekikn", - "turbaszek", - "zhongjiajie", - "ephraimbuddy", - "jhtimmins", - "dstandish", - "xinbinhuang", - "yuqian" - ]'), github.actor) - ) && github.repository == 'apache/airflow' - ) && 'self-hosted' || 'ubuntu-20.04' }} - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} - upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} - kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} - defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }} - kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }} - defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }} - postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }} - defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }} - mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }} - defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }} - helmVersions: ${{ steps.selective-checks.outputs.helm-versions }} - defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }} - kindVersions: ${{ steps.selective-checks.outputs.kind-versions }} - defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }} - testTypes: ${{ steps.selective-checks.outputs.test-types }} - postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }} - mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }} - sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} - run-tests: ${{ steps.selective-checks.outputs.run-tests }} - run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} - image-build: ${{ steps.selective-checks.outputs.image-build }} - docs-build: ${{ steps.selective-checks.outputs.docs-build }} - needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} - needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} - needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} - default-branch: ${{ steps.selective-checks.outputs.default-branch }} - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} - runsOn: ${{ steps.set-runs-on.outputs.runsOn }} - steps: - # Avoid having to specify the runs-on logic every time. We use the custom - # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never - # on the public runners - - name: Set runs-on - id: set-runs-on - run: | - if [[ ${AIRFLOW_SELF_HOSTED_RUNNER} != "" ]]; then - echo "::set-output name=runsOn::\"self-hosted\"" - else - echo "::set-output name=runsOn::\"ubuntu-20.04\"" - fi - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Get information about the PR" - uses: ./.github/actions/get-workflow-origin - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: > - Event: ${{ github.event_name }} - Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - Branch: ${{ github.head_ref }} - Run id: ${{ github.run_id }} - Sha: ${{ github.sha }} - Ref: ${{ github.ref }} - run: printenv - - name: Set wait for image - id: wait-for-image - run: | - if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then - echo "::set-output name=wait-for-image::true" - else - echo "::set-output name=wait-for-image::false" - fi - - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - if: github.event_name == 'pull_request' - - name: Selective checks - id: selective-checks - env: - PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" - run: | - if [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${GITHUB_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi - - test-openapi-client-generation: - timeout-minutes: 10 - name: "Test OpenAPI client generation" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.needs-api-codegen == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Generate client codegen diff" - run: ./scripts/ci/openapi/client_codegen_diff.sh - - test-examples-of-prod-image-building: - timeout-minutes: 60 - name: "Test examples of production image building" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.image-build == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: | - needs.build-info.outputs.waitForImage == 'true' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - - name: "Test examples of PROD image building" - run: ./scripts/ci/images/ci_test_examples_of_prod_image_building.sh - - ci-images: - timeout-minutes: 120 - name: "Wait for CI images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.image-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }} - outputs: - githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - if: needs.build-info.outputs.waitForImage == 'true' - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - if: needs.build-info.outputs.waitForImage == 'true' - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: | - needs.build-info.outputs.waitForImage == 'true' - - name: > - Wait for CI images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - id: wait-for-images - env: - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - # We wait for the images to be available either from the build-ci-image step or from - # "build-images-workflow-run.yml' run as pull_request_target. - # We are utilising single job to wait for all images because this job merely waits - # for the images to be available. - # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'! - # The job will set the output "githubRegistry" - result of auto-detect which registry has - # been used by checking where the image can be downloaded from. - # - run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh - - - static-checks: - timeout-minutes: 30 - name: "Static checks" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - SKIP: "pylint,identity" - MOUNT_SELECTED_LOCAL_SOURCES: "true" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.basic-checks-only == 'false' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Get Python version" - run: "echo \"::set-output name=host-python-version::$(python -c - 'import platform; print(platform.python_version())')\"" - id: host-python-version - - name: "Cache pre-commit local-installation" - uses: actions/cache@v2 - with: - path: ~/.local - key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('setup.py', 'setup.cfg') }}" - restore-keys: "\ -pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" - - name: "Cache pre-commit envs: no-pylint" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}} - - name: "Static checks: except pylint" - run: ./scripts/ci/static_checks/run_static_checks.sh - env: - VERBOSE: false - - # Those checks are run if no image needs to be built for checks. This is for simple changes that - # Do not touch any of the python code or any of the important files that might require building - # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host - static-checks-basic-checks-only: - timeout-minutes: 30 - name: "Static checks: basic checks only" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - SKIP: "build,mypy,flake8,pylint,bats-in-container-tests,identity" - MOUNT_SELECTED_LOCAL_SOURCES: "true" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: needs.build-info.outputs.basic-checks-only == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - - name: > - Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: "Get Python version" - run: "echo \"::set-output name=host-python-version::$(python -c - 'import platform; print(platform.python_version())')\"" - id: host-python-version - - name: "Cache pre-commit local-installation" - uses: actions/cache@v2 - with: - path: ~/.local - key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('setup.py', 'setup.cfg') }}" - restore-keys: "\ -pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" - - name: "Cache pre-commit envs: no-pylint" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}} - - name: "Static checks: basic checks only" - run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}" - env: - VERBOSE: false - - - static-checks-pylint: - timeout-minutes: 30 - name: "Pylint" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - if: needs.build-info.outputs.basic-checks-only == 'false' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - # We want to make sure we have latest sources as only in_container scripts are added - # to the image but we want to static-check all of them - MOUNT_SELECTED_LOCAL_SOURCES: "true" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Get Python version" - run: "echo \"::set-output name=host-python-version::$(python -c - 'import platform; print(platform.python_version())')\"" - id: host-python-version - - name: "Cache pre-commit local-installation" - uses: actions/cache@v2 - with: - path: ~/.local - key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('setup.py', 'setup.cfg') }}" - restore-keys: "\ -pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" - - name: "Cache pre-commit envs - pylint" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-pylint-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-pylint-${{steps.host-python-version.outputs.host-python-version}} - - name: "Static checks: pylint" - run: ./scripts/ci/static_checks/run_static_checks.sh pylint - env: - VERBOSE: false - - docs: - timeout-minutes: 45 - name: "Build docs" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - if: needs.build-info.outputs.docs-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Build docs" - run: ./scripts/ci/docs/ci_docs.sh - - name: Configure AWS credentials - uses: ./.github/actions/configure-aws-credentials - if: > - github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' && - github.event_name == 'push' - with: - aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-central-1 - - name: "Upload documentation to AWS S3" - if: > - github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' && - github.event_name == 'push' - run: aws s3 sync --delete ./docs/_build s3://apache-airflow-docs - - prepare-test-provider-packages-wheel: - timeout-minutes: 40 - name: "Build and test provider packages wheel" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - AIRFLOW_EXTRAS: "all" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - if: needs.build-info.outputs.default-branch == 'master' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Prepare provider documentation" - run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh - - name: "Prepare provider packages: wheel" - run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh - env: - PACKAGE_FORMAT: "wheel" - - name: "Prepare airflow package: wheel" - run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh - env: - PACKAGE_FORMAT: "wheel" - - name: "Install and test provider packages and airflow via wheel files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - INSTALL_AIRFLOW_VERSION: "wheel" - PACKAGE_FORMAT: "wheel" - - name: "Install and test provider packages and airflow on Airflow 2.0 files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - INSTALL_AIRFLOW_VERSION: "2.0.0" - PACKAGE_FORMAT: "wheel" - - prepare-test-provider-packages-sdist: - timeout-minutes: 40 - name: "Build and test provider packages sdist" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - AIRFLOW_EXTRAS: "all" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: "dev" - VERSION_SUFFIX_FOR_SVN: "dev" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - if: needs.build-info.outputs.default-branch == 'master' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Prepare provider packages: sdist" - run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh - env: - PACKAGE_FORMAT: "sdist" - - name: "Prepare airflow package: sdist" - run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh - env: - PACKAGE_FORMAT: "sdist" - - name: "Install and test provider packages and airflow via sdist files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - INSTALL_AIRFLOW_VERSION: "sdist" - PACKAGE_FORMAT: "sdist" - - tests-helm: - timeout-minutes: 20 - name: "Python unit tests for helm chart" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - MOUNT_SELECTED_LOCAL_SOURCES: "true" - TEST_TYPES: "Helm" - BACKEND: "sqlite" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: > - needs.build-info.outputs.needs-helm-tests == 'true' && - (github.repository == 'apache/airflow' || github.event_name != 'schedule') - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-helm - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-helm - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: > - coverage-helm - path: "./files/coverage*.xml" - retention-days: 7 - - tests-postgres: - timeout-minutes: 130 - name: > - Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: - ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: postgres - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - POSTGRES_VERSION: ${{ matrix.postgres-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: > - coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/coverage*.xml" - retention-days: 7 - - tests-mysql: - timeout-minutes: 130 - name: > - MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: mysql - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - MYSQL_VERSION: ${{ matrix.mysql-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/coverage*.xml" - retention-days: 7 - - tests-sqlite: - timeout-minutes: 130 - name: > - Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}} - path: './files/airflow_logs*' - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-sqlite-${{matrix.python-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: coverage-sqlite-${{matrix.python-version}} - path: ./files/coverage*.xml - retention-days: 7 - - tests-quarantined: - timeout-minutes: 60 - name: "Quarantined tests" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - continue-on-error: true - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} - MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} - POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} - TEST_TYPES: "Quarantined" - NUM_RUNS: 10 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Set issue id for master" - if: github.ref == 'refs/heads/master' - run: | - echo "ISSUE_ID=10118" >> $GITHUB_ENV - - name: "Set issue id for v1-10-stable" - if: github.ref == 'refs/heads/v1-10-stable' - run: | - echo "ISSUE_ID=10127" >> $GITHUB_ENV - - name: "Set issue id for v1-10-test" - if: github.ref == 'refs/heads/v1-10-test' - run: | - echo "ISSUE_ID=10128" >> $GITHUB_ENV - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests: Quarantined" - run: ./scripts/ci/testing/ci_run_quarantined_tests.sh - - name: "Upload Quarantine test results" - uses: actions/upload-artifact@v2 - if: always() - with: - name: quarantined_tests - path: "files/test_result-*.xml" - retention-days: 7 - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-quarantined-${{ matrix.backend }} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-quarantined-${{ matrix.backend }} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: coverage-quarantined-${{ matrix.backend }} - path: "./files/coverage*.xml" - retention-days: 7 - - upload-coverage: - timeout-minutes: 15 - name: "Upload coverage" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - continue-on-error: true - needs: - - build-info - - tests-kubernetes - - tests-postgres - - tests-sqlite - - tests-mysql - - tests-quarantined - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Download all artifacts from the current build" - uses: actions/download-artifact@v2 - with: - path: ./coverage-files - - name: "Removes unnecessary artifacts" - run: ls ./coverage-files | grep -v coverage | xargs rm -rf - - name: "Upload all coverage reports to codecov" - uses: ./.github/actions/codecov-action - with: - directory: "./coverage-files" - - prod-images: - timeout-minutes: 120 - name: "Wait for PROD images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - if: needs.build-info.outputs.image-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - outputs: - githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - if: needs.build-info.outputs.waitForImage == 'true' - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - if: needs.build-info.outputs.waitForImage == 'true' - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - if: | - needs.build-info.outputs.waitForImage == 'true' - - name: > - Wait for PROD images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - # We wait for the images to be available either from the build-ci-image step or from - # "build-images-workflow-run.yml' run as pull_request_target. - # We are utilising single job to wait for all images because this job merely waits - # For the images to be available. The test jobs wait for it to complete! - # The job will set the output "githubRegistry" - result of auto-detect which registry has - # been used by checking where the image can be downloaded from. - # - id: wait-for-images - env: - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - - tests-kubernetes: - timeout-minutes: 50 - name: K8s tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, prod-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: postgres - RUN_TESTS: "true" - RUNTIME: "kubernetes" - KUBERNETES_MODE: "image" - KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" - HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetesVersionsListAsString}} - if: needs.build-info.outputs.run-kubernetes-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Get all PROD images" - run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - - name: "Cache virtualenv for kubernetes testing" - uses: actions/cache@v2 - with: - path: ".build/.kubernetes_venv" - key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ --${{needs.build-info.outputs.kubernetesVersionsListAsString}} --${{needs.build-info.outputs.pythonVersionsListAsString}} --${{ hashFiles('setup.py','setup.cfg') }}" - restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ --${{needs.build-info.outputs.kubernetesVersionsListAsString}} --${{needs.build-info.outputs.pythonVersionsListAsString}}" - - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v2 - with: - path: ".build/kubernetes-bin" - key: "kubernetes-binaries --${{ needs.build-info.outputs.defaultKindVersion }}\ --${{ needs.build-info.outputs.defaultHelmVersion }}" - restore-keys: "kubernetes-binaries" - - name: "Kubernetes Tests" - run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh - - name: "Upload KinD logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: > - kind-logs- - path: /tmp/kind_logs_* - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - with: - name: > - coverage-k8s- - path: "./files/coverage*.xml" - retention-days: 7 - - push-prod-images-to-github-registry: - timeout-minutes: 10 - name: "Push PROD images as cache to GitHub Registry" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - static-checks - - static-checks-pylint - - tests-sqlite - - tests-postgres - - tests-mysql - - tests-kubernetes - - prod-images - - docs - if: > - (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test') && - github.event_name != 'schedule' - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: Set push-python-image - id: push-python-image - run: | - if [[ "${REF}" == 'refs/head/master' || "${REF}" == 'refs/head/main' ]]; then - echo "::set-output name=wanted::true" - else - echo "::set-output name=wanted::false" - fi - env: - REF: ${{ github.ref }} - - name: - "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - env: - # Since we are going to push both final image and build image segment, we need to pull the - # build image, in case we are pulling from registry rather than building. - WAIT_FOR_PROD_BUILD_IMAGE: "true" - WAIT_FOR_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}} - - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_push_production_images.sh - env: - PUSH_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}} - - push-ci-images-to-github-registry: - timeout-minutes: 10 - name: "Push CI images as cache to GitHub Registry" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - static-checks - - static-checks-pylint - - tests-sqlite - - tests-postgres - - tests-mysql - - tests-kubernetes - - ci-images - - docs - if: > - (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test') && - github.event_name != 'schedule' - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_push_ci_images.sh - - constraints: - timeout-minutes: 10 - name: "Constraints" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - ci-images - - prod-images - - static-checks - - static-checks-pylint - - tests-sqlite - - tests-mysql - - tests-postgres - - tests-kubernetes - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }} - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} - # Only run it for direct pushes - if: > - github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || - github.ref == 'refs/heads/v2-0-test' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: > - Wait for CI images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh - - name: "Generate constraints with PyPI providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "pypi-providers" - - name: "Generate constraints with source providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "source-providers" - - name: "Generate constraints without providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "no-providers" - - name: "Set constraints branch name" - id: constraints-branch - run: ./scripts/ci/constraints/ci_branch_constraints.sh - - name: Checkout ${{ steps.constraints-branch.outputs.branch }} - uses: actions/checkout@v2 - with: - path: "repo" - ref: ${{ steps.constraints-branch.outputs.branch }} - persist-credentials: false - - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" - run: ./scripts/ci/constraints/ci_commit_constraints.sh - - name: "Push changes" - uses: ./.github/actions/github-push-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ steps.constraints-branch.outputs.branch }} - directory: "repo" - - tag-repo-nightly: - timeout-minutes: 10 - name: "Tag repo nightly" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - docs - - build-info - - static-checks - - static-checks-pylint - - tests-sqlite - - tests-postgres - - tests-mysql - - tests-kubernetes - - constraints - - prepare-test-provider-packages-wheel - - prepare-test-provider-packages-sdist - if: github.event_name == 'schedule' && github.repository == 'apache/airflow' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Tag commit" - run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///') - echo "Tagging ${BRANCH_NAME}" - git tag -f nightly-${BRANCH_NAME} HEAD - - name: "Push tags" - uses: ./.github/actions/github-push-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - tags: true - force: true - branch: master diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 0c35b148556..00000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: "CodeQL" - -on: # yamllint disable-line rule:truthy - push: - branches: [master, main] - schedule: - - cron: '0 2 * * *' - -jobs: - selective-checks: - name: Selective checks - runs-on: ubuntu-20.04 - outputs: - needs-python-scans: ${{ steps.selective-checks.outputs.needs-python-scans }} - needs-javascript-scans: ${{ steps.selective-checks.outputs.needs-javascript-scans }} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: Selective checks - id: selective-checks - env: - EVENT_NAME: ${{ github.event_name }} - TARGET_COMMIT_SHA: ${{ github.sha }} - run: | - if [[ ${EVENT_NAME} == "pull_request" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi - - analyze: - name: Analyze - runs-on: ubuntu-20.04 - needs: [selective-checks] - strategy: - fail-fast: false - matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['python', 'javascript'] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - persist-credentials: false - if: | - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - if: | - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - if: | - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 - if: | - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' diff --git a/.github/workflows/label_when_reviewed.yml b/.github/workflows/label_when_reviewed.yml deleted file mode 100644 index 5095953def1..00000000000 --- a/.github/workflows/label_when_reviewed.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Label when reviewed -on: pull_request_review # yamllint disable-line rule:truthy - -jobs: - - label-when-reviewed: - name: "Label PRs when reviewed" - runs-on: ubuntu-20.04 - steps: - - name: "Do nothing. Only trigger corresponding workflow_run event" - run: echo diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml deleted file mode 100644 index 59bde48284a..00000000000 --- a/.github/workflows/label_when_reviewed_workflow_run.yml +++ /dev/null @@ -1,172 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Label when reviewed workflow run -on: # yamllint disable-line rule:truthy - workflow_run: - workflows: ["Label when reviewed"] - types: ['requested'] -jobs: - - label-when-reviewed: - name: "Label PRs when reviewed workflow run" - runs-on: ubuntu-20.04 - outputs: - labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Get information about the original trigger of the run" - uses: ./.github/actions/get-workflow-origin - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - sourceRunId: ${{ github.event.workflow_run.id }} - - name: Initiate Selective Build check - uses: ./.github/actions/checks-action - id: selective-build-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Selective build check" - status: "in_progress" - sha: ${{ steps.source-run-info.outputs.sourceHeadSha }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Checking selective status of the build in - [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - "} - - name: > - Event: ${{ steps.source-run-info.outputs.sourceEvent }} - Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - Branch: ${{ steps.source-run-info.outputs.sourceHeadBranch }} - Run id: ${{ github.run_id }} - Source Run id: ${{ github.event.workflow_run.id }} - Sha: ${{ github.sha }} - Source Sha: ${{ steps.source-run-info.outputs.sourceHeadSha }} - Merge commit Sha: ${{ steps.source-run-info.outputs.mergeCommitSha }} - Target commit Sha: ${{ steps.source-run-info.outputs.targetCommitSha }} - run: printenv - - name: > - Fetch incoming commit ${{ steps.source-run-info.outputs.targetCommitSha }} with its parent - uses: actions/checkout@v2 - with: - ref: ${{ steps.source-run-info.outputs.targetCommitSha }} - fetch-depth: 2 - persist-credentials: false - # checkout the master version again, to use the right script in master workflow - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Selective checks - id: selective-checks - env: - EVENT_NAME: ${{ steps.source-run-info.outputs.sourceEvent }} - TARGET_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }} - PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }} - run: | - if [[ ${EVENT_NAME} == "pull_request_review" ]]; then - # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" - else - # Run all checks - ./scripts/ci/selective_ci_checks.sh - fi - - name: "Label when approved by committers for PRs that require full tests" - uses: ./.github/actions/label-when-approved-action - id: label-full-test-prs-when-approved-by-commiters - if: > - steps.selective-checks.outputs.run-tests == 'true' && - contains(steps.selective-checks.outputs.test-types, 'Core') - with: - token: ${{ secrets.GITHUB_TOKEN }} - label: 'full tests needed' - require_committers_approval: 'true' - remove_label_when_approval_missing: 'false' - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - comment: > - The PR most likely needs to run full matrix of tests because it modifies parts of the core - of Airflow. However, committers might decide to merge it quickly and take the risk. - If they don't merge it quickly - please rebase it to the latest master at your convenience, - or amend the last commit of the PR, and push it with --force-with-lease. - - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}" - uses: ./.github/actions/checks-action - id: full-test-check - if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true' - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Please rebase or amend, and force push the PR to run full tests" - status: "in_progress" - sha: ${{ steps.source-run-info.outputs.sourceHeadSha }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "The PR likely needs to run all tests! This was determined via selective check in - [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - "} - - name: "Label when approved by committers for PRs that do not require full tests" - uses: ./.github/actions/label-when-approved-action - id: label-simple-test-prs-when-approved-by-commiters - if: > - steps.selective-checks.outputs.run-tests == 'true' && - ! contains(steps.selective-checks.outputs.test-types, 'Core') - with: - token: ${{ secrets.GITHUB_TOKEN }} - label: 'okay to merge' - require_committers_approval: 'true' - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - comment: > - The PR is likely OK to be merged with just subset of tests for default Python and Database - versions without running the full matrix of tests, because it does not modify the core of - Airflow. If the committers decide that the full tests matrix is needed, they will add the label - 'full tests needed'. Then you should rebase to the latest master or amend the last commit - of the PR, and push it with --force-with-lease. - - name: "Label when approved by committers for PRs that do not require tests at all" - uses: ./.github/actions/label-when-approved-action - id: label-no-test-prs-when-approved-by-commiters - if: steps.selective-checks.outputs.run-tests != 'true' - with: - token: ${{ secrets.GITHUB_TOKEN }} - label: 'okay to merge' - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - require_committers_approval: 'true' - comment: > - The PR is likely ready to be merged. No tests are needed as no important environment files, - nor python files were modified by it. However, committers might decide that full test matrix is - needed and add the 'full tests needed' label. Then you should rebase it to the latest master - or amend the last commit of the PR, and push it with --force-with-lease. - - name: Update Selective Build check - uses: ./.github/actions/checks-action - if: always() - with: - token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.selective-build-check.outputs.check_id }} - status: "completed" - sha: ${{ steps.source-run-info.outputs.sourceHeadSha }} - conclusion: ${{ job.status }} - details_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - output: > - {"summary": - "Checking selective status of the build completed in - [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - "} diff --git a/.github/workflows/repo_sync.yml b/.github/workflows/repo_sync.yml deleted file mode 100644 index 76afc192139..00000000000 --- a/.github/workflows/repo_sync.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Force sync master from apache/airflow -on: # yamllint disable-line rule:truthy - workflow_dispatch: -jobs: - repo-sync: - if: github.repository != 'apache/airflow' - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: repo-sync - uses: repo-sync/github-sync@v2 - with: - source_repo: "apache/airflow" - source_branch: "master" - destination_branch: "master" - github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml deleted file mode 100644 index 2a5c4a24e20..00000000000 --- a/.github/workflows/scheduled_quarantined.yml +++ /dev/null @@ -1,107 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Quarantined Build -on: # yamllint disable-line rule:truthy - schedule: - # Run quarantined builds 4 times a day to gather better quarantine stats - - cron: '12 */6 * * *' - -env: - MOUNT_SELECTED_LOCAL_SOURCES: "false" - FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "false" - CHECK_IMAGE_FOR_REBUILD: "true" - SKIP_CHECK_REMOTE_IMAGE: "true" - DB_RESET: "true" - VERBOSE: "true" - UPGRADE_TO_NEWER_DEPENDENCIES: false - PYTHON_MAJOR_MINOR_VERSION: 3.6 - USE_GITHUB_REGISTRY: "true" - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} - # This token is WRITE one - schedule type of events always have the WRITE token - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # This token should not be empty in schedule type of event. - CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }} - # Since we run this build on schedule, it might be that the image has never been pushed - # Because the master merge was cancelled, so we have to rebuild the image for quarantined build - GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" - GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" - GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }} - -jobs: - - tests-quarantined: - timeout-minutes: 80 - name: "Quarantined tests" - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.6] - postgres-version: [9.6] - env: - BACKEND: postgres - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - POSTGRES_VERSION: ${{ matrix.postgres-version }} - RUN_TESTS: "true" - FORCE_TEST_TYPE: Quarantined - NUM_RUNS: 20 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: "Set issue id for master" - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - run: | - echo "ISSUE_ID=10118" >> $GITHUB_ENV - - name: "Set issue id for v1-10-stable" - if: github.ref == 'refs/heads/v1-10-stable' - run: | - echo "ISSUE_ID=10127" >> $GITHUB_ENV - - name: "Set issue id for v1-10-test" - if: github.ref == 'refs/heads/v1-10-test' - run: | - echo "ISSUE_ID=10128" >> $GITHUB_ENV - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - env: - GITHUB_REGISTRY: ${{ steps.determine-github-registry.outputs.githubRegistry }} - - name: "Tests" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - uses: actions/upload-artifact@v2 - name: Upload Quarantine test results - if: always() - with: - name: 'quarantined_tests' - path: 'files/test_result-*.xml' - retention-days: 7 - - uses: actions/upload-artifact@v2 - name: Upload airflow logs - if: always() - with: - name: airflow-logs-quarantined-${{matrix.python-version}}-${{ matrix.postgres-version }} - path: './files/airflow_logs*' - retention-days: 7
