This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fix-generated-dependencies-script in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 895c72d6e2a4f7749f4058aad8b0f3ce8d138371 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Dec 18 17:38:03 2023 +0100 Remove additional generation of dependencies when building CI images When generated dependencies are not properly updated, we had a special step where the dependencies were generated "just in case" before CI image was built, because otherwise building the CI image could have failed with strange "failed because of conflicting dependencies" without a clue what was the root cause. However, the pre-commit did not return error exit code - because for the pre-commit, it is enough that a file is modified during pre-commit to fail the pre-commit in general. That had a nasty side effect because the built CI image actually already contained properly generated dependencies (by this step), and it did not properly detected cases where the ones in the repository were added manually and not generated with pre-commit. This PR fixes it - instead of generating and building such image in CI it will now fail the CI image building step but with clear instructions what to do. The CI job step uses now regular breeze command rather than running the script manually but also the script returns error code in case the generated dependencies have been updated. --- .github/actions/build-ci-images/action.yml | 8 ++------ generated/provider_dependencies.json | 2 +- scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py | 8 +++++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml index c437835a4d..c400b88974 100644 --- a/.github/actions/build-ci-images/action.yml +++ b/.github/actions/build-ci-images/action.yml @@ -23,13 +23,9 @@ runs: steps: - name: "Install Breeze" uses: ./.github/actions/breeze - - name: "Regenerate dependencies" - # This is done in case some someone updated provider dependencies and did not generate - # dependencies - in which case build image might fail because of lack of new dependencies + - name: "Check if dependencies are properly regenerated" shell: bash - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py + run: breeze static-checks --type update-providers-dependencies --all-files if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}" shell: bash diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 780009c536..b755f6c839 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -418,7 +418,7 @@ "asgiref>=3.5.2", "gcloud-aio-auth>=4.0.0,<5.0.0", "gcloud-aio-bigquery>=6.1.2", - "gcloud-aio-storage", + "gcloud-aio-storage>=9.0.0", "gcsfs>=2023.10.0", "google-ads>=22.1.0", "google-api-core>=2.11.0", diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index ada9f884be..a3056df3a1 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -213,11 +213,12 @@ if __name__ == "__main__": DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n") if os.environ.get("CI"): console.print() - console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}") + console.print(f"[info]There is a need to regenerate {DEPENDENCIES_JSON_FILE_PATH}") console.print( - f"[yellow]You will need to run breeze locally and commit " - f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)}!\n" + f"[red]You need to run the following command locally and commit generated " + f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)} file:\n" ) + console.print("breeze static-checks --type update-providers-dependencies --all-files") console.print() else: console.print() @@ -227,6 +228,7 @@ if __name__ == "__main__": ) console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}") console.print() + sys.exit(1) else: console.print( "[green]No need to regenerate dependencies!\n[/]"
