This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a450b81c51 Move constraints generation to separate job in CI workflow
(#34990)
a450b81c51 is described below
commit a450b81c5120519214c2eb3fbf55e729fd847d91
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Oct 18 00:13:23 2023 +0200
Move constraints generation to separate job in CI workflow (#34990)
We generate constraints for preview right after images are build, in
order to be able to see the output and to diagnose which dependencies
have been updated later in the process.
So far we were doing it in CI workflow and When constraints generation
failed however (for example because pip backtracking takes a log ot
time), it faied build image workflow and did not allow tests to
complete.
This PR extracts constraint generation to CI separate job which
does not block tests from running in case constraints generation
fails or times out.
It also moves out the steps from the composite action which allows
to better see which step failed and allows to see the constraints
used in each job more easily.
---
.github/actions/build-ci-images/action.yml | 24 --------
.github/workflows/ci.yml | 72 +++++++++++++++++++++---
Dockerfile.ci | 7 ++-
scripts/in_container/_in_container_utils.sh | 3 +-
scripts/in_container/run_generate_constraints.sh | 1 +
setup.py | 2 +-
6 files changed, 74 insertions(+), 35 deletions(-)
diff --git a/.github/actions/build-ci-images/action.yml
b/.github/actions/build-ci-images/action.yml
index 81825f9a79..e93b2f4e71 100644
--- a/.github/actions/build-ci-images/action.yml
+++ b/.github/actions/build-ci-images/action.yml
@@ -34,30 +34,6 @@ runs:
- name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{
env.PYTHON_VERSIONS }}"
shell: bash
run: breeze ci-image build --push --tag-as-latest --run-in-parallel
--upgrade-on-failure
- - name: "Generate source constraints"
- shell: bash
- run: >
- breeze release-management generate-constraints --run-in-parallel
- --airflow-constraints-mode constraints-source-providers
-# - name: "Generate PyPI constraints"
-# shell: bash
-# run: >
-# breeze release-management generate-constraints --run-in-parallel
-# --airflow-constraints-mode constraints
-# if: ${{ inputs.build-provider-packages != 'true' }}
- - name: "Print dependency upgrade summary"
- shell: bash
- run: |
- for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
- echo "Summarizing Python $PYTHON_VERSION"
- cat "files/constraints-${PYTHON_VERSION}/*.md" >>
$GITHUB_STEP_SUMMARY || true
- done
- - name: "Upload constraint artifacts"
- uses: actions/upload-artifact@v3
- with:
- name: constraints
- path: ./files/constraints-*/constraints-*.txt
- retention-days: 7
- name: "Fix ownership"
shell: bash
run: breeze ci fix-ownership
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca44ad3199..325865f741 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -326,6 +326,61 @@ jobs:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
BUILD_TIMEOUT_MINUTES: 70
+ preview-constraints:
+ permissions:
+ contents: read
+ timeout-minutes: 20
+ continue-on-error: true
+ name: >
+ Preview constraints
+ ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+ runs-on: "${{needs.build-info.outputs.runs-on}}"
+ needs: [build-info, build-ci-images]
+ if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
+ env:
+ RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
+ PYTHON_VERSIONS:
${{needs.build-info.outputs.all-python-versions-list-as-string}}
+ DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ submodules: recursive
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ run: breeze ci-image pull --run-in-parallel --tag-as-latest
+ - name: "Source constraints"
+ shell: bash
+ run: >
+ breeze release-management generate-constraints --run-in-parallel
+ --airflow-constraints-mode constraints-source-providers || true
+ - name: "PyPI constraints"
+ shell: bash
+ timeout-minutes: 25
+ run: >
+ breeze release-management generate-constraints --run-in-parallel
+ --airflow-constraints-mode constraints || true
+ - name: "Dependency upgrade summary"
+ shell: bash
+ run: |
+ for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
+ echo "Summarizing Python $PYTHON_VERSION"
+ cat "files/constraints-${PYTHON_VERSION}"/*.md >>
$GITHUB_STEP_SUMMARY || true
+ done
+ - name: "Upload constraint artifacts"
+ uses: actions/upload-artifact@v3
+ with:
+ name: constraints
+ path: ./files/constraints-*/constraints-*.txt
+ retention-days: 7
+ - name: "Fix ownership"
+ run: breeze ci fix-ownership
+ if: always()
+
build-prod-images:
timeout-minutes: 80
name: >
@@ -1662,6 +1717,7 @@ jobs:
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+ PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
@@ -1677,16 +1733,18 @@ jobs:
run: breeze ci-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- - name: "Generate constraints"
+ - name: "Generate source constraints"
+ run: |
+ breeze release-management generate-constraints \
+ --run-in-parallel --airflow-constraints-mode
constraints-source-providers
+ - name: "Generate no-providers constraints"
run: |
- breeze release-management generate-constraints --run-in-parallel \
- --airflow-constraints-mode constraints-source-providers
breeze release-management generate-constraints \
--run-in-parallel --airflow-constraints-mode
constraints-no-providers
-# breeze release-management generate-constraints \
-# --run-in-parallel --airflow-constraints-mode constraints
- env:
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
+ - name: "Generate PyPi constraints"
+ run: |
+ breeze release-management generate-constraints \
+ --run-in-parallel --airflow-constraints-mode constraints
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 3e8fca1a58..ec35aab734 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1435,8 +1435,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
# Those are additional constraints that are needed for some extras but we do
not want to
# force them on the main Airflow package. Currently we need no extra limits as
PIP 23.1+ has much better
# dependency resolution and we do not need to limit the versions of the
dependencies
-# aiobotocore is limited temporarily until it stops backtracking pip
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
+#
+# Without grpcio-status limit, pip gets into very long backtracking
+# We should attempt to remove it in the future
+#
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="grpcio-status>=1.55.0"
ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
ARG VERSION_SUFFIX_FOR_PYPI=""
diff --git a/scripts/in_container/_in_container_utils.sh
b/scripts/in_container/_in_container_utils.sh
index c962856827..d1cf4d6a5b 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -333,9 +333,10 @@ function
install_all_providers_from_pypi_with_eager_upgrade() {
# Installing it with Airflow makes sure that the version of package that
matches current
# Airflow requirements will be used.
# shellcheck disable=SC2086
+ set -x
pip install ".[${NO_PROVIDERS_EXTRAS}]" "${packages_to_install[@]}"
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} \
--upgrade --upgrade-strategy eager
-
+ set +x
}
function install_all_provider_packages_from_wheels() {
diff --git a/scripts/in_container/run_generate_constraints.sh
b/scripts/in_container/run_generate_constraints.sh
index d3b4ad20e0..2c969c5db5 100755
--- a/scripts/in_container/run_generate_constraints.sh
+++ b/scripts/in_container/run_generate_constraints.sh
@@ -152,5 +152,6 @@ echo
echo "Constraints error markdown generated in ${CONSTRAINTS_MARKDOWN_DIFF}"
echo
+ls "${CONSTRAINTS_MARKDOWN_DIFF}"
exit 0
diff --git a/setup.py b/setup.py
index d3aee366f9..cd1a56626d 100644
--- a/setup.py
+++ b/setup.py
@@ -1042,4 +1042,4 @@ def do_setup() -> None:
if __name__ == "__main__":
- do_setup() # comment
+ do_setup() # comment to trigger upgrade to newer dependencies
