This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b14f16c5ccc Always use shared cache, no matter if we are using uv or
not (#50923)
b14f16c5ccc is described below
commit b14f16c5ccc8ea37dc5d8d2d37546fa28409232b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu May 22 12:03:59 2025 +0200
Always use shared cache, no matter if we are using uv or not (#50923)
Another attempt to fix pyspark issues. It turns out that we are not
passing the `use_uv` flag in tests, so the cache has not been
actually shared between paralllel containers - we change it
in the way that the uv cache is shared regardless if use-uv is passed
---
Dockerfile.ci | 4 +++-
dev/breeze/src/airflow_breeze/params/shell_params.py | 4 ++--
scripts/ci/docker-compose/{ci-uv-tests.yml => ci-tests.yml} | 4 ++--
scripts/docker/entrypoint_ci.sh | 4 +++-
4 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 038f6aebe8d..c1c1002762f 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1020,7 +1020,9 @@ function determine_airflow_to_use() {
echo
# Use uv run to install necessary dependencies automatically
# in the future we will be able to use uv sync when `uv.lock` is
supported
- uv run
/opt/airflow/scripts/in_container/install_development_dependencies.py \
+ # for the use in parallel runs in docker containers--no-cache is
needed - otherwise there is
+ # possibility of overriding temporary environments by multiple
parallel processes
+ uv run --no-cache
/opt/airflow/scripts/in_container/install_development_dependencies.py \
--constraint
https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
# Some packages might leave legacy typing module which causes test
issues
# shellcheck disable=SC2086
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 9894ae2cc43..4a7f952ad0e 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -351,8 +351,8 @@ class ShellParams:
self.add_docker_in_docker(compose_file_list)
compose_file_list.extend(backend_files)
compose_file_list.append(DOCKER_COMPOSE_DIR / "files.yml")
- if os.environ.get("CI", "false") == "true" and self.use_uv:
- compose_file_list.append(DOCKER_COMPOSE_DIR / "ci-uv-tests.yml")
+ if os.environ.get("CI", "false") == "true":
+ compose_file_list.append(DOCKER_COMPOSE_DIR / "ci-tests.yml")
if self.use_airflow_version is not None and self.mount_sources not in
USE_AIRFLOW_MOUNT_SOURCES:
get_console().print(
diff --git a/scripts/ci/docker-compose/ci-uv-tests.yml
b/scripts/ci/docker-compose/ci-tests.yml
similarity index 93%
rename from scripts/ci/docker-compose/ci-uv-tests.yml
rename to scripts/ci/docker-compose/ci-tests.yml
index e293a8ceef9..3e962a05532 100644
--- a/scripts/ci/docker-compose/ci-uv-tests.yml
+++ b/scripts/ci/docker-compose/ci-tests.yml
@@ -19,5 +19,5 @@ services:
airflow:
volumes:
# We should be ok with sharing the cache between the builds - now that
we are using uv
- # The cache should be safe to share between parallel builds as UV is
build to support it.
- - /mnt/.cache:/root/.cache
+ # The cache should be safe to share between parallel builds.
+ - /mnt/.cache/uv:/root/.cache/uv
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 1c2a3e5efc4..9dcb3ee6583 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -232,7 +232,9 @@ function determine_airflow_to_use() {
echo
# Use uv run to install necessary dependencies automatically
# in the future we will be able to use uv sync when `uv.lock` is
supported
- uv run
/opt/airflow/scripts/in_container/install_development_dependencies.py \
+ # for the use in parallel runs in docker containers--no-cache is
needed - otherwise there is
+ # possibility of overriding temporary environments by multiple
parallel processes
+ uv run --no-cache
/opt/airflow/scripts/in_container/install_development_dependencies.py \
--constraint
https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
# Some packages might leave legacy typing module which causes test
issues
# shellcheck disable=SC2086
