This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch build-common-io-from-sources in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c9a0cd4ad7cc47cc4e9f8f3840be5fa31a89a76a Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 27 15:48:43 2023 +0100 Add feature to build "chicken-egg" packages from sources When we build on ci a pre-release version of provider, and we want to include packages that have >= CURRENTLY_RELEASED_VERSION we have to make sure that those packages are built from sources during building of PROD image. Otherwise they will not be installable on CURRENT_VERSION.dev0, CURRENT_VERSION.rc* etc. Until we "Actually" release a provider we should have a way to build such provider from sources. This is the CI version of it, once we have it working we can also apply it to the workflow that releases images to dockerhub. --- .github/actions/build-prod-images/action.yml | 12 +++++ .github/workflows/build-images.yml | 2 + .github/workflows/ci.yml | 14 ++++++ .../commands/release_management_commands.py | 24 ++++++--- .../commands/release_management_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 2 + dev/breeze/src/airflow_breeze/utils/packages.py | 4 -- .../src/airflow_breeze/utils/selective_checks.py | 5 ++ ...put_release-management_generate-constraints.svg | 58 ++++++++++++++-------- ...put_release-management_generate-constraints.txt | 2 +- scripts/in_container/_in_container_utils.sh | 16 ++++-- 11 files changed, 102 insertions(+), 38 deletions(-) diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 5fdbb795c4..f038234087 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -22,6 +22,9 @@ inputs: build-provider-packages: description: 'Whether to build provider packages from sources' required: true + chicken-egg-providers: + description: 'List of chicken-egg provider packages to build from sources' + required: true runs: using: "composite" steps: @@ -41,6 +44,15 @@ runs: --package-list-file ./airflow/providers/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: ${{ inputs.build-provider-packages == 'true' }} + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }} + if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }} - name: "Prepare airflow package" shell: bash run: > diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index b29d49de17..82cc6e5987 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -74,6 +74,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || github.event.pull_request.head.sha || github.sha @@ -293,6 +294,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74093010c0..9e8fdd9223 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -474,6 +475,7 @@ jobs: RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + VERSION_SUFFIX_FOR_PYPI: "dev0" if: needs.build-info.outputs.ci-image-build == 'true' steps: - name: Cleanup repo @@ -502,6 +504,16 @@ jobs: run: > breeze release-management generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 + ${{ needs.build-info.outputs.chicken-egg-providers }} + if: ${{ needs.build-info.outputs.chicken-egg-providers }} - name: "PyPI constraints" shell: bash timeout-minutes: 25 @@ -1608,6 +1620,7 @@ jobs: if: needs.build-info.outputs.in-workflow-build == 'true' with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -1647,6 +1660,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index bbcf525e77..ed6545f67b 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -684,21 +684,30 @@ def run_generate_constraints_in_parallel( @option_image_tag_for_running @option_debug_release_management @option_airflow_constraints_mode_ci [email protected]( + "--chicken-egg-providers", + default="", + help="List of chicken-egg provider packages - " + "those that have airflow_version >= current_version and should " + "be installed in CI from locally built packages with >= current_version.dev0 ", + envvar="CHICKEN_EGG_PROVIDERS", +) @option_github_repository @option_verbose @option_dry_run @option_answer def generate_constraints( + airflow_constraints_mode: str, + debug: bool, + debug_resources: bool, + github_repository: str, + image_tag: str | None, + parallelism: int, python: str, + python_versions: str, run_in_parallel: bool, - parallelism: int, skip_cleanup: bool, - debug_resources: bool, - python_versions: str, - image_tag: str | None, - debug: bool, - airflow_constraints_mode: str, - github_repository: str, + chicken_egg_providers: str, ): perform_environment_checks() check_remote_ghcr_io_commands() @@ -742,6 +751,7 @@ def generate_constraints( python=python, github_repository=github_repository, airflow_constraints_mode=airflow_constraints_mode, + chicken_egg_providers=chicken_egg_providers, ) for python in python_version_list ] diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index e27ccc3dd6..ff0c9cdcd0 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -134,6 +134,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "--image-tag", "--python", "--airflow-constraints-mode", + "--chicken-egg-providers", "--debug", "--github-repository", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 32f6a111ce..abfe2c4ddd 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -118,6 +118,7 @@ class ShellParams: builder: str = "autodetect" celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False + chicken_egg_providers: str = "" collect_only: bool = False database_isolation: bool = False db_reset: bool = False @@ -435,6 +436,7 @@ class ShellParams: _set_var(_env, "BREEZE", "true") _set_var(_env, "BREEZE_INIT_COMMAND", None, "") _set_var(_env, "CELERY_FLOWER", self.celery_flower) + _set_var(_env, "CHICKEN_EGG_PROVIDERS", self.chicken_egg_providers) _set_var(_env, "CI", None, "false") _set_var(_env, "CI_BUILD_ID", None, "0") _set_var(_env, "CI_EVENT_TYPE", None, "pull_request") diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 3d59643dbc..5518fdebd7 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -56,10 +56,6 @@ LONG_PROVIDERS_PREFIX = "apache-airflow-providers-" # TODO: use single source of truth for those # for now we need to keep them in sync with the ones in setup.py PREINSTALLED_PROVIDERS = [ - # Until we cut off the 2.8.0 branch and bump current airflow version to 2.9.0, we should - # Keep common.io commented out in order ot be able to generate PyPI constraints because - # The version from PyPI has requirement of apache-airflow>=2.8.0 - # "common.io", "common.sql", "ftp", "http", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 0a751565ef..eba3023a6c 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1019,3 +1019,8 @@ class SelectiveChecks: @cached_property def has_migrations(self) -> bool: return any([file.startswith("airflow/migrations/") for file in self._files]) + + @cached_property + def chicken_egg_providers(self) -> str: + """Space separated list of providers with chicken-egg problem and should be built from sources.""" + return "common.io" diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg index 9801441959..0fd9c471c2 100644 --- a/images/breeze/output_release-management_generate-constraints.svg +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 855.1999999999999" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 952.8" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-generate-constraints-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="804.1999999999999" /> + <rect x="0" y="0" width="1463.0" height="901.8" /> </clipPath> <clipPath id="breeze-release-management-generate-constraints-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -141,9 +141,21 @@ <clipPath id="breeze-release-management-generate-constraints-line-31"> <rect x="0" y="757.9" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-32"> + <rect x="0" y="782.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-33"> + <rect x="0" y="806.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-34"> + <rect x="0" y="831.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-35"> + <rect x="0" y="855.5" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="853.2" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="950.8" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -168,24 +180,28 @@ </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8" textLength="97.6" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-release-mana [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &# [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="73.2" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="410.4" textLength="85.4" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-18)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="459.2" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-18)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="459.2" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="48.8" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="146.4" clip-path="url(#breeze-relea [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="532.4" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-21)">[default: 4; 1<=x<=8]                 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="556.8" textLength="85.4" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="581.2" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-23)">[default: 3.8 3.9 3.10 3.11]              [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="61" clip-path="url(#breeze-re [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="630" textLength="73.2" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-26)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-27)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="678.8" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="678.8" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="97.6" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="48.8" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="752" textLength="85.4" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="776.4" textLength="61" clip-path="url(#breeze-re [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-32)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages  [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  & [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                     [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="73.2" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="85.4" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-22)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="556.8" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-22)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="556.8" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="581.2" textLength="48.8" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="146.4" clip-path="url(#breeze [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="630" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-25)">[default: 4; 1<=x<=8]                 [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="654.4" textLength="85.4" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="678.8" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-27)">[default: 3.8 3.9 3.10 3.11]              [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="61" clip-path="url(#breeze-re [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="73.2" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-31)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="776.4" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="776.4" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="800.8" textLength="97.6" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="825.2" textLength="48.8" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="849.6" textLength="85.4" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="874" textLength="61" clip-path="url(#breeze-release- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="898.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-36)"> </text> </g> </g> diff --git a/images/breeze/output_release-management_generate-constraints.txt b/images/breeze/output_release-management_generate-constraints.txt index c714ab26a1..4ac25d8890 100644 --- a/images/breeze/output_release-management_generate-constraints.txt +++ b/images/breeze/output_release-management_generate-constraints.txt @@ -1 +1 @@ -01aef235b11e59ed7f10c970a5cdaba7 +a2ba8a1912a7e9eca347cc616e717dba diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index a3e09e0f1e..c316710e4b 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -233,13 +233,19 @@ function install_all_providers_from_pypi_with_eager_upgrade() { local packages_to_install=() local provider_package local res + local chicken_egg_prefixes + chicken_egg_prefixes="" + for chicken_egg_provider in ${CHICKEN_EGG_PROVIDERS} + do + chicken_egg_prefixes="${chicken_egg_prefixes} apache-airflow-providers-${chicken_egg_provider//./-}" + done for provider_package in ${ALL_PROVIDERS_PACKAGES} do - # Remove common.io provider in main branch until we cut-off v2-8-test branch and change - # version in main to 2.9.0 - otherwise we won't be able to generate PyPI constraints as - # released common-io provider has apache-airflow>2.8.0 as dependency and we cannot install - # the provider from PyPI - if [[ ${provider_package} == "apache-airflow-providers-common-io" ]]; then + if [[ "${chicken_egg_prefixes}" == *"${provider_package}"* ]]; then + # add the provider prepared in dist folder where chicken - egg problem is mitigated + package_file="./dist/${provider_package//./_}*.whl" + packages_to_install+=( "${package_file}" ) + echo "Added ${package_file} from dist folder as this is a chicken-egg package ${COLOR_GREEN}OK${COLOR_RESET}" continue fi echo -n "Checking if ${provider_package} is available in PyPI: "
