This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch move-early-constrints-generation-to-separate-steps in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 49c8cda813fd9097df34b40238680af9e06fed15 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Oct 16 16:49:08 2023 +0200 Move constraints generation to separate job in CI workflow We generate constraints for preview right after images are build, in order to be able to see the output and to diagnose which dependencies have been updated later in the process. So far we were doing it in CI workflow and When constraints generation failed however (for example because pip backtracking takes a log ot time), it faied build image workflow and did not allow tests to complete. This PR extracts constraint generation to CI separate job which does not block tests from running in case constraints generation fails or times out. It also moves out the steps from the composite action which allows to better see which step failed and allows to see the constraints used in each job more easily. --- .github/actions/build-ci-images/action.yml | 24 -------- .github/workflows/ci.yml | 72 +++++++++++++++++++++--- airflow/providers/google/provider.yaml | 1 + generated/provider_dependencies.json | 1 + scripts/in_container/_in_container_utils.sh | 3 +- scripts/in_container/run_generate_constraints.sh | 1 + setup.py | 2 +- 7 files changed, 71 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml index 81825f9a79..e93b2f4e71 100644 --- a/.github/actions/build-ci-images/action.yml +++ b/.github/actions/build-ci-images/action.yml @@ -34,30 +34,6 @@ runs: - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}" shell: bash run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure - - name: "Generate source constraints" - shell: bash - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers -# - name: "Generate PyPI constraints" -# shell: bash -# run: > -# breeze release-management generate-constraints --run-in-parallel -# --airflow-constraints-mode constraints -# if: ${{ inputs.build-provider-packages != 'true' }} - - name: "Print dependency upgrade summary" - shell: bash - run: | - for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do - echo "Summarizing Python $PYTHON_VERSION" - cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true - done - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v3 - with: - name: constraints - path: ./files/constraints-*/constraints-*.txt - retention-days: 7 - name: "Fix ownership" shell: bash run: breeze ci fix-ownership diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca44ad3199..d2d4243a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,6 +326,61 @@ jobs: DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} BUILD_TIMEOUT_MINUTES: 70 + preview-constraints: + permissions: + contents: read + timeout-minutes: 20 + continue-on-error: true + name: > + Preview constraints + ${{needs.build-info.outputs.all-python-versions-list-as-string}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, build-ci-images] + if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' + env: + RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" + PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} + DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + run: breeze ci-image pull --run-in-parallel --tag-as-latest + - name: "Source constraints" + shell: bash + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-source-providers || true + - name: "PyPI constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints || true + - name: "Dependency upgrade summary" + shell: bash + run: | + for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do + echo "Summarizing Python $PYTHON_VERSION" + cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true + done + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@v3 + with: + name: constraints + path: ./files/constraints-*/constraints-*.txt + retention-days: 7 + - name: "Fix ownership" + run: breeze ci fix-ownership + if: always() + build-prod-images: timeout-minutes: 80 name: > @@ -1662,6 +1717,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' steps: - name: Cleanup repo @@ -1677,16 +1733,18 @@ jobs: run: breeze ci-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - - name: "Generate constraints" + - name: "Generate source constraints" + run: | + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints-source-providers + - name: "Generate no-providers constraints" run: | - breeze release-management generate-constraints --run-in-parallel \ - --airflow-constraints-mode constraints-source-providers breeze release-management generate-constraints \ --run-in-parallel --airflow-constraints-mode constraints-no-providers -# breeze release-management generate-constraints \ -# --run-in-parallel --airflow-constraints-mode constraints - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} + - name: "Generate PyPi constraints" + run: | + breeze release-management generate-constraints \ + --run-in-parallel --airflow-constraints-mode constraints - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 5ceb155495..591bc5c68f 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -123,6 +123,7 @@ dependencies: - google-cloud-workflows>=1.10.0 - google-cloud-run>=0.9.0 - google-cloud-batch>=0.13.0 + - grpcio-status>=1.55.0 - grpcio-gcp>=0.2.2 - httpx - json-merge-patch>=0.2 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 1a8c9c3a89..a607c123c1 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -449,6 +449,7 @@ "google-cloud-vision>=3.4.0", "google-cloud-workflows>=1.10.0", "grpcio-gcp>=0.2.2", + "grpcio-status>=1.55.0", "httpx", "json-merge-patch>=0.2", "looker-sdk>=22.2.0", diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index c962856827..d1cf4d6a5b 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -333,9 +333,10 @@ function install_all_providers_from_pypi_with_eager_upgrade() { # Installing it with Airflow makes sure that the version of package that matches current # Airflow requirements will be used. # shellcheck disable=SC2086 + set -x pip install ".[${NO_PROVIDERS_EXTRAS}]" "${packages_to_install[@]}" ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} \ --upgrade --upgrade-strategy eager - + set +x } function install_all_provider_packages_from_wheels() { diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh index d3b4ad20e0..2c969c5db5 100755 --- a/scripts/in_container/run_generate_constraints.sh +++ b/scripts/in_container/run_generate_constraints.sh @@ -152,5 +152,6 @@ echo echo "Constraints error markdown generated in ${CONSTRAINTS_MARKDOWN_DIFF}" echo +ls "${CONSTRAINTS_MARKDOWN_DIFF}" exit 0 diff --git a/setup.py b/setup.py index d3aee366f9..cd1a56626d 100644 --- a/setup.py +++ b/setup.py @@ -1042,4 +1042,4 @@ def do_setup() -> None: if __name__ == "__main__": - do_setup() # comment + do_setup() # comment to trigger upgrade to newer dependencies
