This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch build-common-io-from-sources in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 46f304df67d228bfad80d59bd47eff69cfa47cbf Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Nov 27 15:48:43 2023 +0100 Add feature to build "chicken-egg" packages from sources When we build on ci a pre-release version of provider, and we want to include packages that have >= CURRENTLY_RELEASED_VERSION we have to make sure that those packages are built from sources during building of PROD image. Otherwise they will not be installable on CURRENT_VERSION.dev0, CURRENT_VERSION.rc* etc. Until we "Actually" release a provider we should have a way to build such provider from sources. This is the CI version of it, once we have it working we can also apply it to the workflow that releases images to dockerhub. --- .github/actions/build-prod-images/action.yml | 12 ++++++++++++ .github/workflows/build-images.yml | 2 ++ .github/workflows/ci.yml | 3 +++ airflow/providers/installed_providers.txt | 1 + dev/breeze/src/airflow_breeze/utils/packages.py | 4 ---- dev/breeze/src/airflow_breeze/utils/selective_checks.py | 10 ++++++++++ scripts/in_container/_in_container_utils.sh | 7 ------- 7 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 5fdbb795c4..2e928e8bb3 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -22,6 +22,9 @@ inputs: build-provider-packages: description: 'Whether to build provider packages from sources' required: true + chicken-egg-providers: + description: 'Whether to build chicken-egg packages' + required: true runs: using: "composite" steps: @@ -41,6 +44,15 @@ runs: --package-list-file ./airflow/providers/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: ${{ inputs.build-provider-packages == 'true' }} + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }} + if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }} - name: "Prepare airflow package" shell: bash run: > diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index b29d49de17..82cc6e5987 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -74,6 +74,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || github.event.pull_request.head.sha || github.sha @@ -293,6 +294,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53e4da6916..72ab1dbd8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -1608,6 +1609,7 @@ jobs: if: needs.build-info.outputs.in-workflow-build == 'true' with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -1647,6 +1649,7 @@ jobs: uses: ./.github/actions/build-prod-images with: build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/airflow/providers/installed_providers.txt b/airflow/providers/installed_providers.txt index bd32056d5f..0d9b03a55e 100644 --- a/airflow/providers/installed_providers.txt +++ b/airflow/providers/installed_providers.txt @@ -1,6 +1,7 @@ amazon celery cncf.kubernetes +common.io common.sql daskexecutor docker diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 3d59643dbc..5518fdebd7 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -56,10 +56,6 @@ LONG_PROVIDERS_PREFIX = "apache-airflow-providers-" # TODO: use single source of truth for those # for now we need to keep them in sync with the ones in setup.py PREINSTALLED_PROVIDERS = [ - # Until we cut off the 2.8.0 branch and bump current airflow version to 2.9.0, we should - # Keep common.io commented out in order ot be able to generate PyPI constraints because - # The version from PyPI has requirement of apache-airflow>=2.8.0 - # "common.io", "common.sql", "ftp", "http", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 0a751565ef..b9d194e67c 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1019,3 +1019,13 @@ class SelectiveChecks: @cached_property def has_migrations(self) -> bool: return any([file.startswith("airflow/migrations/") for file in self._files]) + + @cached_property + def chicken_egg_providers(self) -> str: + """Space separated list of providers with chicken-egg problem and should be built from sources.""" + if self._default_branch != "main": + # TODO: This is temporary only - until we have packages that have >= UPCOMING_VERSION of Airflow + # Because packages released in PyPI have >= UPCOMING_VERSION rather than >= UPCOMING_VERSION.dev0 + # Once the release is done, we should remove this and build package from PyPI + return "common.io" + return "" diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index a3e09e0f1e..f78e044420 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -235,13 +235,6 @@ function install_all_providers_from_pypi_with_eager_upgrade() { local res for provider_package in ${ALL_PROVIDERS_PACKAGES} do - # Remove common.io provider in main branch until we cut-off v2-8-test branch and change - # version in main to 2.9.0 - otherwise we won't be able to generate PyPI constraints as - # released common-io provider has apache-airflow>2.8.0 as dependency and we cannot install - # the provider from PyPI - if [[ ${provider_package} == "apache-airflow-providers-common-io" ]]; then - continue - fi echo -n "Checking if ${provider_package} is available in PyPI: " res=$(curl --head -s -o /dev/null -w "%{http_code}" "https://pypi.org/project/${provider_package}/";) if [[ ${res} == "200" ]]; then
