This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch add-better-dependencies-diagnostics in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a943509ee6524295e2f478e19aea98edfa423a6c Author: Jarek Potiuk <[email protected]> AuthorDate: Thu Jun 30 18:10:05 2022 +0200 Better diagnostics in case someone modified the dependencies file The "provider_dependencies.json" file gets automatically generated by pre-commit and in case it cannot be json-parsed, it will break building the CI image. This is a bit chicken-egg because on CI the image is needed to run pre-commits that could warn the user this is the case (and the image fails build with a bit cryptic message). This PR improves the diagnostics: * it runs the pre-commit check before image building which will fix the generated file (and will let the build run - only to fail at the latest static-checks step * it prints warning to the user seeing pre-commit error to not modify the file manually but let pre-commit do the job. --- .github/workflows/build-images.yml | 4 ++++ .github/workflows/ci.yml | 6 ++++++ scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 8903d87da3..a79640667a 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -224,6 +224,8 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: breeze static-checks --type update-providers-dependencies --all-files || true - name: > Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.allPythonVersionsListAsString }} @@ -316,6 +318,8 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: breeze static-checks --type update-providers-dependencies --all-files || true - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 768e3ad15d..4ebd3370d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,6 +332,9 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: breeze static-checks --type update-providers-dependencies --all-files || true + if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: > Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.allPythonVersionsListAsString }} @@ -402,6 +405,9 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: breeze static-checks --type update-providers-dependencies --all-files || true + if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py index af2471ad62..8ee81908f4 100755 --- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py @@ -195,6 +195,10 @@ if __name__ == '__main__': console.print() sys.exit(1) DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n") + console.print( + f"[yellow]If you see changes to the {DEPENDENCIES_JSON_FILE_PATH} file - " + f"do not modify the file manually. Let pre-commit do the job!" + ) console.print() console.print("[green]Verification complete! Success!\n") console.print(f"Written {DEPENDENCIES_JSON_FILE_PATH}")
