This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8a0252d419cd2ae94557cbdc026513804698a894 Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Dec 1 01:09:48 2023 +0100 Add feature to build "chicken-egg" packages from sources (#35890) When we build on ci a pre-release version of provider, and we want to include packages that have >= CURRENTLY_RELEASED_VERSION we have to make sure that those packages are built from sources during building of PROD image. Otherwise they will not be installable on CURRENT_VERSION.dev0, CURRENT_VERSION.rc* etc. Until we "Actually" release a provider we should have a way to build such provider from sources. This is the CI version of it, once we have it working we can also apply it to the workflow that releases images to dockerhub. (cherry picked from commit cf052dc64f00e851427a41a34ffe576fd39be51b) --- .github/actions/build-prod-images/action.yml | 12 +++++ .github/workflows/build-images.yml | 2 + .github/workflows/ci.yml | 15 ++++++ airflow/providers/amazon/provider.yaml | 2 +- airflow/providers/google/provider.yaml | 2 +- airflow/providers/microsoft/azure/provider.yaml | 2 +- .../commands/release_management_commands.py | 25 +++++++--- .../commands/release_management_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 2 + dev/breeze/src/airflow_breeze/utils/packages.py | 4 -- .../src/airflow_breeze/utils/selective_checks.py | 5 ++ docs/apache-airflow-providers-google/index.rst | 2 +- .../index.rst | 2 +- generated/provider_dependencies.json | 4 +- ...put_release-management_generate-constraints.svg | 58 ++++++++++++++-------- ...put_release-management_generate-constraints.txt | 2 +- scripts/in_container/_in_container_utils.sh | 23 +++++++-- setup.cfg | 2 +- setup.py | 2 +- 19 files changed, 120 insertions(+), 47 deletions(-) diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 5fdbb795c4..f038234087 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -22,6 +22,9 @@ inputs: build-provider-packages: description: 'Whether to build provider packages from sources' required: true + chicken-egg-providers: + description: 'List of chicken-egg provider packages to build from sources' + required: true runs: using: "composite" steps: @@ -41,6 +44,15 @@ runs: --package-list-file ./airflow/providers/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: ${{ inputs.build-provider-packages == 'true' }} + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }} + if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }} - name: "Prepare airflow package" shell: bash run: > diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index b29d49de17..82cc6e5987 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -74,6 +74,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || github.event.pull_request.head.sha || github.sha @@ -293,6 +294,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53e4da6916..09d672546a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -474,6 +475,7 @@ jobs: RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + VERSION_SUFFIX_FOR_PYPI: "dev0" if: needs.build-info.outputs.ci-image-build == 'true' steps: - name: Cleanup repo @@ -502,12 +504,23 @@ jobs: run: > breeze release-management generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 + ${{ needs.build-info.outputs.chicken-egg-providers }} + if: needs.build-info.outputs.chicken-egg-providers != '' - name: "PyPI constraints" shell: bash timeout-minutes: 25 run: > breeze release-management generate-constraints --run-in-parallel --airflow-constraints-mode constraints + --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" - name: "Dependency upgrade summary" shell: bash run: | @@ -1608,6 +1621,7 @@ jobs: if: needs.build-info.outputs.in-workflow-build == 'true' with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -1647,6 +1661,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index d6869683d0..798043abe8 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -753,7 +753,7 @@ additional-extras: - apache-airflow-providers-cncf-kubernetes>=7.2.0 - name: s3fs dependencies: - - s3fs>=2023.9.2 + - s3fs>=2023.10.0 - name: python3-saml dependencies: - python3-saml>=1.16.0 diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 86a03ff900..7e6f69349c 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -87,7 +87,7 @@ dependencies: - gcloud-aio-auth>=4.0.0,<5.0.0 - gcloud-aio-bigquery>=6.1.2 - gcloud-aio-storage - - gcsfs>=2023.9.2 + - gcsfs>=2023.10.0 - google-ads>=22.1.0 - google-api-core>=2.11.0 - google-api-python-client>=1.6.0 diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 4f2308803d..186719a9af 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -70,7 +70,7 @@ versions: dependencies: - apache-airflow>=2.5.0 - - adlfs>=2023.9.2 + - adlfs>=2023.10.0 - azure-batch>=8.0.0 - azure-cosmos>=4.0.0 - azure-mgmt-cosmosdb diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index bbcf525e77..5aa1f65bf1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -684,21 +684,30 @@ def run_generate_constraints_in_parallel( @option_image_tag_for_running @option_debug_release_management @option_airflow_constraints_mode_ci [email protected]( + "--chicken-egg-providers", + default="", + help="List of chicken-egg provider packages - " + "those that have airflow_version >= current_version and should " + "be installed in CI from locally built packages with >= current_version.dev0 ", + envvar="CHICKEN_EGG_PROVIDERS", +) @option_github_repository @option_verbose @option_dry_run @option_answer def generate_constraints( + airflow_constraints_mode: str, + debug: bool, + debug_resources: bool, + github_repository: str, + image_tag: str | None, + parallelism: int, python: str, + python_versions: str, run_in_parallel: bool, - parallelism: int, skip_cleanup: bool, - debug_resources: bool, - python_versions: str, - image_tag: str | None, - debug: bool, - airflow_constraints_mode: str, - github_repository: str, + chicken_egg_providers: str, ): perform_environment_checks() check_remote_ghcr_io_commands() @@ -742,6 +751,7 @@ def generate_constraints( python=python, github_repository=github_repository, airflow_constraints_mode=airflow_constraints_mode, + chicken_egg_providers=chicken_egg_providers, ) for python in python_version_list ] @@ -762,6 +772,7 @@ def generate_constraints( skip_image_upgrade_check=True, quiet=True, airflow_constraints_mode=airflow_constraints_mode, + chicken_egg_providers=chicken_egg_providers, ) return_code, info = run_generate_constraints( shell_params=shell_params, diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index e27ccc3dd6..ff0c9cdcd0 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -134,6 +134,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "--image-tag", "--python", "--airflow-constraints-mode", + "--chicken-egg-providers", "--debug", "--github-repository", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index a314075d29..2ed8215f91 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -118,6 +118,7 @@ class ShellParams: builder: str = "autodetect" celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False + chicken_egg_providers: str = "" collect_only: bool = False database_isolation: bool = False db_reset: bool = False @@ -435,6 +436,7 @@ class ShellParams: _set_var(_env, "BREEZE", "true") _set_var(_env, "BREEZE_INIT_COMMAND", None, "") _set_var(_env, "CELERY_FLOWER", self.celery_flower) + _set_var(_env, "CHICKEN_EGG_PROVIDERS", self.chicken_egg_providers) _set_var(_env, "CI", None, "false") _set_var(_env, "CI_BUILD_ID", None, "0") _set_var(_env, "CI_EVENT_TYPE", None, "pull_request") diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 3d59643dbc..5518fdebd7 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -56,10 +56,6 @@ LONG_PROVIDERS_PREFIX = "apache-airflow-providers-" # TODO: use single source of truth for those # for now we need to keep them in sync with the ones in setup.py PREINSTALLED_PROVIDERS = [ - # Until we cut off the 2.8.0 branch and bump current airflow version to 2.9.0, we should - # Keep common.io commented out in order ot be able to generate PyPI constraints because - # The version from PyPI has requirement of apache-airflow>=2.8.0 - # "common.io", "common.sql", "ftp", "http", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 0a751565ef..eba3023a6c 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1019,3 +1019,8 @@ class SelectiveChecks: @cached_property def has_migrations(self) -> bool: return any([file.startswith("airflow/migrations/") for file in self._files]) + + @cached_property + def chicken_egg_providers(self) -> str: + """Space separated list of providers with chicken-egg problem and should be built from sources.""" + return "common.io" diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index d642d19ee2..b9defcd150 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -116,7 +116,7 @@ PIP package Version required ``gcloud-aio-auth`` ``>=4.0.0,<5.0.0`` ``gcloud-aio-bigquery`` ``>=6.1.2`` ``gcloud-aio-storage`` -``gcsfs`` ``>=2023.9.2`` +``gcsfs`` ``>=2023.10.0`` ``google-ads`` ``>=22.1.0`` ``google-api-core`` ``>=2.11.0`` ``google-api-python-client`` ``>=1.6.0`` diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index 95a23f3957..cc1467e64f 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -106,7 +106,7 @@ The minimum Apache Airflow version supported by this provider package is ``2.5.0 PIP package Version required ================================ ================== ``apache-airflow`` ``>=2.5.0`` -``adlfs`` ``>=2023.9.2`` +``adlfs`` ``>=2023.10.0`` ``azure-batch`` ``>=8.0.0`` ``azure-cosmos`` ``>=4.0.0`` ``azure-mgmt-cosmosdb`` diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 0ff1e0b32e..3764acbf30 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -425,7 +425,7 @@ "gcloud-aio-auth>=4.0.0,<5.0.0", "gcloud-aio-bigquery>=6.1.2", "gcloud-aio-storage", - "gcsfs>=2023.9.2", + "gcsfs>=2023.10.0", "google-ads>=22.1.0", "google-api-core>=2.11.0", "google-api-python-client>=1.6.0", @@ -567,7 +567,7 @@ "microsoft.azure": { "deps": [ "adal>=1.2.7", - "adlfs>=2023.9.2", + "adlfs>=2023.10.0", "apache-airflow>=2.5.0", "azure-batch>=8.0.0", "azure-cosmos>=4.0.0", diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg index 9801441959..0fd9c471c2 100644 --- a/images/breeze/output_release-management_generate-constraints.svg +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 855.1999999999999" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 952.8" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-generate-constraints-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="804.1999999999999" /> + <rect x="0" y="0" width="1463.0" height="901.8" /> </clipPath> <clipPath id="breeze-release-management-generate-constraints-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -141,9 +141,21 @@ <clipPath id="breeze-release-management-generate-constraints-line-31"> <rect x="0" y="757.9" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-32"> + <rect x="0" y="782.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-33"> + <rect x="0" y="806.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-34"> + <rect x="0" y="831.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-generate-constraints-line-35"> + <rect x="0" y="855.5" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="853.2" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="950.8" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -168,24 +180,28 @@ </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8" textLength="97.6" clip-path="url(#breeze- [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-release-mana [...] </text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &# [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="73.2" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="410.4" textLength="85.4" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-18)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="459.2" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-18)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="459.2" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="48.8" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="146.4" clip-path="url(#breeze-relea [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="532.4" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-21)">[default: 4; 1<=x<=8]                 [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="556.8" textLength="85.4" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="581.2" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-23)">[default: 3.8 3.9 3.10 3.11]              [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="61" clip-path="url(#breeze-re [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="630" textLength="73.2" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-26)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)"> -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-27)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="678.8" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="678.8" textLeng [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="97.6" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="48.8" clip-path="url(#breeze- [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="752" textLength="85.4" clip-path="url(#breeze-releas [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="776.4" textLength="61" clip-path="url(#breeze-re [...] -</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-32)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages  [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  & [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                     [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="73.2" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="85.4" clip-path="url(#breeze-releas [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-22)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="556.8" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-22)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="556.8" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="581.2" textLength="48.8" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="146.4" clip-path="url(#breeze [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="630" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-25)">[default: 4; 1<=x<=8]                 [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="654.4" textLength="85.4" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="678.8" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-27)">[default: 3.8 3.9 3.10 3.11]              [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="61" clip-path="url(#breeze-re [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="73.2" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)"> +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-31)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="776.4" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="776.4" textLeng [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="800.8" textLength="97.6" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="825.2" textLength="48.8" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="849.6" textLength="85.4" clip-path="url(#breeze- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="874" textLength="61" clip-path="url(#breeze-release- [...] +</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="898.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-36)"> </text> </g> </g> diff --git a/images/breeze/output_release-management_generate-constraints.txt b/images/breeze/output_release-management_generate-constraints.txt index c714ab26a1..4ac25d8890 100644 --- a/images/breeze/output_release-management_generate-constraints.txt +++ b/images/breeze/output_release-management_generate-constraints.txt @@ -1 +1 @@ -01aef235b11e59ed7f10c970a5cdaba7 +a2ba8a1912a7e9eca347cc616e717dba diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index a3e09e0f1e..a321469dc1 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -233,13 +233,26 @@ function install_all_providers_from_pypi_with_eager_upgrade() { local packages_to_install=() local provider_package local res + local chicken_egg_prefixes + chicken_egg_prefixes="" + if [[ ${CHICKEN_EGG_PROVIDERS=} != "" ]]; then + echo "${COLOR_BLUE}Finding providers to install from dist: ${CHICKEN_EGG_PROVIDERS}${COLOR_RESET}" + for chicken_egg_provider in ${CHICKEN_EGG_PROVIDERS} + do + chicken_egg_prefixes="${chicken_egg_prefixes} apache-airflow-providers-${chicken_egg_provider//./-}" + done + echo "${COLOR_BLUE}Chicken egg prefixes: ${chicken_egg_prefixes}${COLOR_RESET}" + ls /dist/ + fi for provider_package in ${ALL_PROVIDERS_PACKAGES} do - # Remove common.io provider in main branch until we cut-off v2-8-test branch and change - # version in main to 2.9.0 - otherwise we won't be able to generate PyPI constraints as - # released common-io provider has apache-airflow>2.8.0 as dependency and we cannot install - # the provider from PyPI - if [[ ${provider_package} == "apache-airflow-providers-common-io" ]]; then + if [[ "${chicken_egg_prefixes}" == *"${provider_package}"* ]]; then + # add the provider prepared in dist folder where chicken - egg problem is mitigated + for file in /dist/"${provider_package//-/_}"*.whl + do + packages_to_install+=( "${file}" ) + echo "Added ${file} from dist folder as this is a chicken-egg package ${COLOR_GREEN}OK${COLOR_RESET}" + done continue fi echo -n "Checking if ${provider_package} is available in PyPI: " diff --git a/setup.cfg b/setup.cfg index 47bcddf04c..916031e305 100644 --- a/setup.cfg +++ b/setup.cfg @@ -106,7 +106,7 @@ install_requires = flask-login>=0.6.2 flask-session>=0.4.0 flask-wtf>=0.15 - fsspec>=2023.9.2 + fsspec>=2023.10.0 google-re2>=1.0 graphviz>=0.12 gunicorn>=20.1.0 diff --git a/setup.py b/setup.py index 9d71ea6194..eacde7a499 100644 --- a/setup.py +++ b/setup.py @@ -524,7 +524,7 @@ aiobotocore = [ s3fs = [ # This is required for support of S3 file system which uses aiobotocore # which can have a conflict with boto3 as mentioned above - "s3fs>=2023.9.2", + "s3fs>=2023.10.0", ] saml = [
