This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d9c973bda1d14474edb714796366bc0060e4cbbc Author: Ash Berlin-Taylor <[email protected]> AuthorDate: Mon Jan 11 10:38:15 2021 +0000 Run "third party" github actions from submodules instead (#13514) Rather than having to mirror all the repos we can instead use git submodules to pull in the third party actions we want to use - with recent(ish) changes in review for submodules on GitHub we still get the same "review/audit" visibility for changes, but this way we don't have to either "pollute" our repo with the actions code, nor do we have to maintain a fork of the third party action. (cherry picked from commit f115983550833b1516ce58e66b49fe4fb1a0eff7) --- .github/actions/cancel-workflow-runs | 1 + .github/actions/checks-action | 1 + .github/actions/codecov-action | 1 + .github/actions/configure-aws-credentials | 1 + .github/actions/get-workflow-origin | 1 + .github/actions/github-push-action | 1 + .github/actions/label-when-approved-action | 1 + .github/workflows/build-images-workflow-run.yml | 57 ++++++++++++++-------- .github/workflows/ci.yml | 22 ++++++--- .../workflows/label_when_reviewed_workflow_run.yml | 24 ++++----- .gitmodules | 21 ++++++++ .pre-commit-config.yaml | 2 +- 12 files changed, 94 insertions(+), 39 deletions(-) diff --git a/.github/actions/cancel-workflow-runs b/.github/actions/cancel-workflow-runs new file mode 160000 index 0000000..953e057 --- /dev/null +++ b/.github/actions/cancel-workflow-runs @@ -0,0 +1 @@ +Subproject commit 953e057dc81d3458935a18d1184c386b0f6b5738 diff --git a/.github/actions/checks-action b/.github/actions/checks-action new file mode 160000 index 0000000..9f02872 --- /dev/null +++ b/.github/actions/checks-action @@ -0,0 +1 @@ +Subproject commit 9f02872da71b6f558c6a6f190f925dde5e4d8798 diff --git a/.github/actions/codecov-action b/.github/actions/codecov-action new file mode 160000 index 0000000..1fc7722 --- /dev/null +++ b/.github/actions/codecov-action @@ -0,0 +1 @@ +Subproject commit 1fc7722ded4708880a5aea49f2bfafb9336f0c8d diff --git a/.github/actions/configure-aws-credentials b/.github/actions/configure-aws-credentials new file mode 160000 index 0000000..e97d7fb --- /dev/null +++ b/.github/actions/configure-aws-credentials @@ -0,0 +1 @@ +Subproject commit e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b diff --git a/.github/actions/get-workflow-origin b/.github/actions/get-workflow-origin new file mode 160000 index 0000000..588cc14 --- /dev/null +++ b/.github/actions/get-workflow-origin @@ -0,0 +1 @@ +Subproject commit 588cc14f9f1cdf1b8be3db816855e96422204fec diff --git a/.github/actions/github-push-action b/.github/actions/github-push-action new file mode 160000 index 0000000..40bf560 --- /dev/null +++ b/.github/actions/github-push-action @@ -0,0 +1 @@ +Subproject commit 40bf560936a8022e68a3c00e7d2abefaf01305a6 diff --git a/.github/actions/label-when-approved-action b/.github/actions/label-when-approved-action new file mode 160000 index 0000000..4c5190f --- /dev/null +++ b/.github/actions/label-when-approved-action @@ -0,0 +1 @@ +Subproject commit 4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index bed5dfc..5c85cb4 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -56,14 +56,19 @@ jobs: cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }} buildImages: ${{ steps.build-images.outputs.buildImages }} steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Get information about the original trigger of the run" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 + uses: ./.github/actions/get-workflow-origin id: source-run-info with: token: ${{ secrets.GITHUB_TOKEN }} sourceRunId: ${{ github.event.workflow_run.id }} - name: "Cancel duplicated 'CI Build' runs" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: allDuplicates @@ -80,7 +85,7 @@ jobs: # trick ¯\_(ツ)_/¯. We name the build-info job appropriately # and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the # current Event/Repo/Branch combination. - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: namedJobs token: ${{ secrets.GITHUB_TOKEN }} @@ -96,7 +101,7 @@ jobs: # We also produce list of canceled "CI Build' runs as output, so that we # can cancel all the matching "Build Images" workflow runs in the two following steps. # Yeah. Adding to the complexity ¯\_(ツ)_/¯. - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs id: cancel-failed with: token: ${{ secrets.GITHUB_TOKEN }} @@ -129,14 +134,14 @@ jobs: # it to cancel any jobs that have matching names containing Source Run Id: # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯. if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]' - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: namedJobs token: ${{ secrets.GITHUB_TOKEN }} notifyPRCancel: true jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }} - name: "Cancel duplicated 'CodeQL' runs" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs id: cancel with: token: ${{ secrets.GITHUB_TOKEN }} @@ -156,7 +161,7 @@ jobs: # trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match # all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those # This might cancel own run, so this is the last step in the job - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: cancelMode: allDuplicatedNamedJobs token: ${{ secrets.GITHUB_TOKEN }} @@ -307,8 +312,19 @@ jobs: else echo "::set-output name=proceed::false" fi + - name: > + Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder + to use ci/scripts from there. + uses: actions/checkout@v2 + with: + path: "main-airflow" + ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" + persist-credentials: false + submodules: recursive + if: steps.defaults.outputs.proceed == 'true' - name: Initiate Github Checks for Building image - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + # Use the submodule from main, not the PR branch + uses: ./main-airflow/.github/actions/checks-action id: build-image-check with: token: ${{ secrets.GITHUB_TOKEN }} @@ -322,15 +338,6 @@ jobs: [Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details" } if: steps.defaults.outputs.proceed == 'true' - - name: > - Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder - to use ci/scripts from there. - uses: actions/checkout@v2 - with: - path: "main-airflow" - ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" - persist-credentials: false - if: steps.defaults.outputs.proceed == 'true' - name: "Setup python" uses: actions/setup-python@v2 with: @@ -363,7 +370,7 @@ jobs: run: ./scripts/ci/images/ci_push_production_images.sh if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true' - name: Update Github Checks for Building image with status - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./main-airflow/.github/actions/checks-action if: always() && steps.defaults.outputs.proceed == 'true' with: token: ${{ secrets.GITHUB_TOKEN }} @@ -384,8 +391,13 @@ jobs: if: cancelled() needs: [build-images] steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: self @@ -399,8 +411,13 @@ jobs: if: failure() needs: [build-images] steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Canceling the 'CI Build' source workflow in case of failure!" - uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7 + uses: ./.github/actions/cancel-workflow-runs with: token: ${{ secrets.GITHUB_TOKEN }} cancelMode: self diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 728dfb0..e87564d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,15 +98,16 @@ jobs: pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} steps: - - name: "Get information about the PR" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive + - name: "Get information about the PR" + uses: ./.github/actions/get-workflow-origin + id: source-run-info + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: > Event: ${{ github.event_name }} Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} @@ -296,6 +297,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Build docs" @@ -617,6 +619,11 @@ jobs: - tests-mysql - tests-quarantined steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Download all artifacts from the current build" uses: actions/download-artifact@v2 with: @@ -624,7 +631,7 @@ jobs: - name: "Removes unnecessary artifacts" run: ls ./coverage-files | grep -v coverage | xargs rm -rf - name: "Upload all coverage reports to codecov" - uses: apache/airflow-codecov-action@1fc7722ded4708880a5aea49f2bfafb9336f0c8d # v1.1.1 + uses: ./.github/actions/codecov-action with: directory: "./coverage-files" @@ -905,6 +912,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh @@ -921,7 +929,7 @@ jobs: - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" run: ./scripts/ci/constraints/ci_commit_constraints.sh - name: "Push changes" - uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0 + uses: ./.github/actions/github-push-action with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.constraints-branch.outputs.branch }} diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml index 4400bae..1ed50dd 100644 --- a/.github/workflows/label_when_reviewed_workflow_run.yml +++ b/.github/workflows/label_when_reviewed_workflow_run.yml @@ -29,14 +29,19 @@ jobs: outputs: labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }} steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - name: "Get information about the original trigger of the run" - uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 + uses: ./.github/actions/get-workflow-origin id: source-run-info with: token: ${{ secrets.GITHUB_TOKEN }} sourceRunId: ${{ github.event.workflow_run.id }} - name: Initiate Selective Build check - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action id: selective-build-check with: token: ${{ secrets.GITHUB_TOKEN }} @@ -49,10 +54,6 @@ jobs: "Checking selective status of the build in [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) "} - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - name: > Event: ${{ steps.source-run-info.outputs.sourceEvent }} Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} @@ -76,6 +77,7 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + submodules: recursive - name: Selective checks id: selective-checks env: @@ -91,7 +93,7 @@ jobs: ./scripts/ci/selective_ci_checks.sh fi - name: "Label when approved by committers for PRs that require full tests" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-full-test-prs-when-approved-by-commiters if: > steps.selective-checks.outputs.run-tests == 'true' && @@ -107,7 +109,7 @@ jobs: If they don't merge it quickly - please rebase it to the latest master at your convenience, or amend the last commit of the PR, and push it with --force-with-lease. - name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}" - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action id: full-test-check if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true' with: @@ -122,7 +124,7 @@ jobs: [the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) "} - name: "Label when approved by committers for PRs that do not require full tests" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-simple-test-prs-when-approved-by-commiters if: > steps.selective-checks.outputs.run-tests == 'true' && @@ -139,7 +141,7 @@ jobs: 'full tests needed'. Then you should rebase to the latest master or amend the last commit of the PR, and push it with --force-with-lease. - name: "Label when approved by committers for PRs that do not require tests at all" - uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3 + uses: ./.github/actions/label-when-approved-action id: label-no-test-prs-when-approved-by-commiters if: steps.selective-checks.outputs.run-tests != 'true' with: @@ -153,7 +155,7 @@ jobs: needed and add the 'full tests needed' label. Then you should rebase it to the latest master or amend the last commit of the PR, and push it with --force-with-lease. - name: Update Selective Build check - uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0 + uses: ./.github/actions/checks-action if: always() with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1779ddd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,21 @@ +[submodule ".github/actions/get-workflow-origin"] + path = .github/actions/get-workflow-origin + url = https://github.com/potiuk/get-workflow-origin +[submodule ".github/actions/cancel-workflow-runs"] + path = .github/actions/cancel-workflow-runs + url = https://github.com/potiuk/cancel-workflow-runs +[submodule ".github/actions/checks-action"] + path = .github/actions/checks-action + url = https://github.com/LouisBrunner/checks-action +[submodule ".github/actions/configure-aws-credentials"] + path = .github/actions/configure-aws-credentials + url = https://github.com/aws-actions/configure-aws-credentials +[submodule ".github/actions/codecov-action"] + path = .github/actions/codecov-action + url = https://github.com/codecov/codecov-action +[submodule ".github/actions/github-push-action"] + path = .github/actions/github-push-action + url = https://github.com/ad-m/github-push-action +[submodule ".github/actions/label-when-approved-action"] + path = .github/actions/label-when-approved-action + url = https://github.com/TobKed/label-when-approved-action diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e6207d..a98b0ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: rev: v1.1.9 hooks: - id: forbid-tabs - exclude: ^docs/Makefile$|^clients/gen/go.sh + exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$ - id: insert-license name: Add license for all SQL files files: \.sql$
