This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 444c6b0f5ced95263dea804a955fc877271efaf9 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Feb 11 20:15:25 2024 +0100 Follow up after moving hatch_build.py to source root (#37341) Turns out that #37340 was not enough to make airflow build from git url - we also need to move provider lists. (cherry picked from commit 1e4a2d5c765b0f98cab3017389b3a03d86e38649) --- .dockerignore | 2 ++ .github/actions/build-prod-images/action.yml | 2 +- .github/workflows/ci.yml | 2 +- Dockerfile.ci | 2 +- ...re_installed_providers.txt => airflow_pre_installed_providers.txt | 0 dev/README_RELEASE_AIRFLOW.md | 2 +- dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 1 + dev/refresh_images.sh | 2 +- docker_tests/test_prod_image.py | 5 ++--- hatch_build.py | 3 +-- ...age_installed_providers.txt => prod_image_installed_providers.txt | 0 scripts/ci/docker-compose/local.yml | 3 +++ scripts/ci/pre_commit/pre_commit_sort_installed_providers.py | 5 ++--- 13 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.dockerignore b/.dockerignore index ae3496d802..e7431de857 100644 --- a/.dockerignore +++ b/.dockerignore @@ -54,6 +54,8 @@ !empty !Dockerfile !hatch_build.py +!prod_image_installed_providers.txt +!airflow_pre_installed_providers.txt # This folder is for you if you want to add any packages to the docker context when you build your own # docker image. most of other files and any new folder you add will be excluded by default diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 141a2cf9b1..47f3934566 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: > breeze release-management prepare-provider-packages - --package-list-file ./dev/prod_image_installed_providers.txt + --package-list-file ./prod_image_installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: ${{ inputs.build-provider-packages == 'true' }} - name: "Prepare chicken-eggs provider packages" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d1a8bfb2e..5dac5ce49f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2441,7 +2441,7 @@ jobs: - name: "Prepare providers packages for PROD build" run: > breeze release-management prepare-provider-packages - --package-list-file ./dev/prod_image_installed_providers.txt + --package-list-file ./prod_image_installed_providers.txt --package-format wheel env: VERSION_SUFFIX_FOR_PYPI: "dev0" diff --git a/Dockerfile.ci b/Dockerfile.ci index 73b7bd5313..5d593b31b6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1178,7 +1178,7 @@ COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY generated/* ${AIRFLOW_SOURCES}/generated/ COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ COPY LICENSE ${AIRFLOW_SOURCES}/LICENSE -COPY dev/airflow_pre_installed_providers.txt ${AIRFLOW_SOURCES}/dev/airflow_pre_installed_providers.txt +COPY airflow_pre_installed_providers.txt ${AIRFLOW_SOURCES}/ COPY hatch_build.py ${AIRFLOW_SOURCES}/ COPY --from=scripts install_airflow.sh /scripts/docker/ diff --git a/dev/airflow_pre_installed_providers.txt b/airflow_pre_installed_providers.txt similarity index 100% rename from dev/airflow_pre_installed_providers.txt rename to airflow_pre_installed_providers.txt diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index efcd9b1cd6..2748543cf2 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -778,7 +778,7 @@ git push -f apache constraints-X.Y.Z that have >= ``X.Y.0`` in the corresponding provider.yaml file. -3. In case the provider should also be installed in the image (it is part of ``dev/prod_image_installed_providers.txt``) +3. In case the provider should also be installed in the image (it is part of ``prod_image_installed_providers.txt``) it should also be added at this moment to ``Dockerfile`` to the list of default extras in the line with ``AIRFLOW_EXTRAS``: ```Dockerfile diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 3307866454..b6f2cc37c2 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -98,6 +98,7 @@ VOLUMES_FOR_SELECTED_MOUNTS = [ ("kubernetes_tests", "/opt/airflow/kubernetes_tests"), ("docker_tests", "/opt/airflow/docker_tests"), ("chart", "/opt/airflow/chart"), + ("hatch_build.py", "/opt/airflow/hatch_build.py"), ] diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index a6d4e9a06f..26bf00dd72 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -36,7 +36,7 @@ breeze ci-image build \ rm -fv ./dist/* ./docker-context-files/* breeze release-management prepare-provider-packages \ - --package-list-file ./dev/prod_image_installed_providers.txt \ + --package-list-file ./prod_image_installed_providers.txt \ --package-format wheel \ --version-suffix-for-pypi dev0 diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py index 4f6a93de42..ac56cc6eff 100644 --- a/docker_tests/test_prod_image.py +++ b/docker_tests/test_prod_image.py @@ -33,9 +33,8 @@ from docker_tests.docker_tests_utils import ( run_python_in_docker, ) -DEV_DIR_PATH = SOURCE_ROOT / "dev" -AIRFLOW_PRE_INSTALLED_PROVIDERS_FILE_PATH = DEV_DIR_PATH / "airflow_pre_installed_providers.txt" -PROD_IMAGE_PROVIDERS_FILE_PATH = DEV_DIR_PATH / "prod_image_installed_providers.txt" +AIRFLOW_PRE_INSTALLED_PROVIDERS_FILE_PATH = SOURCE_ROOT / "airflow_pre_installed_providers.txt" +PROD_IMAGE_PROVIDERS_FILE_PATH = SOURCE_ROOT / "prod_image_installed_providers.txt" AIRFLOW_ROOT_PATH = Path(__file__).parents[2].resolve() SLIM_IMAGE_PROVIDERS = [ f"apache-airflow-providers-{provider_id.replace('.','-')}" diff --git a/hatch_build.py b/hatch_build.py index 844bd39dca..2d6a4ed861 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -34,8 +34,7 @@ log.setLevel(log_level) AIRFLOW_ROOT_PATH = Path(__file__).parent.resolve() GENERATED_PROVIDERS_DEPENDENCIES_FILE = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json" -DEV_DIR_PATH = AIRFLOW_ROOT_PATH / "dev" -PREINSTALLED_PROVIDERS_FILE = DEV_DIR_PATH / "airflow_pre_installed_providers.txt" +PREINSTALLED_PROVIDERS_FILE = AIRFLOW_ROOT_PATH / "airflow_pre_installed_providers.txt" DEPENDENCIES = json.loads(GENERATED_PROVIDERS_DEPENDENCIES_FILE.read_text()) PREINSTALLED_PROVIDER_IDS = [ package.strip() diff --git a/dev/prod_image_installed_providers.txt b/prod_image_installed_providers.txt similarity index 100% rename from dev/prod_image_installed_providers.txt rename to prod_image_installed_providers.txt diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 07ec8a89c6..2a55d8733c 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -109,4 +109,7 @@ services: - type: bind source: ../../../chart target: /opt/airflow/chart + - type: bind + source: ../../../hatch_build.py + target: /opt/airflow/hatch_build.py # END automatically generated volumes from VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py diff --git a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py index fb735569a3..897fddf2ef 100755 --- a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py +++ b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py @@ -27,7 +27,6 @@ if __name__ not in ("__main__", "__mp_main__"): AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -DEV_DIR_PATH = AIRFLOW_SOURCES / "dev" def stable_sort(x): @@ -45,7 +44,7 @@ def sort_file(path: Path): if __name__ == "__main__": - prod_image_installed_providers_path = DEV_DIR_PATH / "prod_image_installed_providers.txt" - airflow_pre_installed_providers_path = DEV_DIR_PATH / "airflow_pre_installed_providers.txt" + prod_image_installed_providers_path = AIRFLOW_SOURCES / "prod_image_installed_providers.txt" + airflow_pre_installed_providers_path = AIRFLOW_SOURCES / "airflow_pre_installed_providers.txt" sort_file(prod_image_installed_providers_path) sort_file(airflow_pre_installed_providers_path)
