This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1934c8b86e Rename build arg for upgrade-to-newer-dependencies (#38637)
1934c8b86e is described below
commit 1934c8b86eea80b9f3957f198ac5ee6ce342f1f5
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Apr 2 10:26:37 2024 +0200
Rename build arg for upgrade-to-newer-dependencies (#38637)
The "UPGRADE_TO_NEWER_DEPENDENCIES" build arg was a bit misleading
as it was not a true/false value but rather "random hash"/"false".
This PR makes it a bit more explicit:
* the arg is named UPGRADE_INVALIDATION_STRING now
* it's default is "" not false and all conditions are set to check
for != or == "".
The inputs/outputs in CI job remain as they were - "true/false" but
this PR makes it clearer.
---
.github/workflows/prod-image-build.yml | 2 +-
Dockerfile | 11 +-
Dockerfile.ci | 9 +-
dev/breeze/doc/ci/02_images.md | 2 +-
dev/breeze/doc/ci/04_selective_checks.md | 2 +-
.../src/airflow_breeze/params/build_ci_params.py | 2 +-
docs/docker-stack/build-arg-ref.rst | 120 ++++++++++-----------
scripts/ci/docker-compose/devcontainer.env | 2 +-
scripts/docker/install_additional_dependencies.sh | 3 +-
scripts/docker/install_airflow.sh | 6 +-
10 files changed, 78 insertions(+), 81 deletions(-)
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 6ea5135c6a..6b2bd17672 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -103,7 +103,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
upgrade-to-newer-dependencies:
- description: "Whether to attempt to upgrade image to newer
dependencies (false/RANDOM_VALUE)"
+ description: "Whether to attempt to upgrade image to newer
dependencies (true/false)"
required: true
type: string
chicken-egg-providers:
diff --git a/Dockerfile b/Dockerfile
index 0ce05f4bad..bd8b7bb38f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -903,7 +903,7 @@ function install_airflow() {
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra
'postgres' installations were therefore omitted.${COLOR_RESET}"
fi
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Remove airflow and all provider packages installed
before potentially${COLOR_RESET}"
echo
@@ -961,13 +961,12 @@ COPY <<"EOF" /install_additional_dependencies.sh
#!/usr/bin/env bash
set -euo pipefail
-: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}"
: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
function install_additional_dependencies() {
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Installing additional dependencies while upgrading
to newer dependencies${COLOR_RESET}"
echo
@@ -1489,7 +1488,7 @@ ARG AIRFLOW_VERSION_SPECIFICATION
# set to "." and "/opt/airflow" respectively.
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
# By default we do not upgrade to latest dependencies
-ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
+ARG UPGRADE_INVALIDATION_STRING=""
ARG AIRFLOW_SOURCES_FROM
ARG AIRFLOW_SOURCES_TO
@@ -1533,7 +1532,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
AIRFLOW_HOME=${AIRFLOW_HOME} \
AIRFLOW_UID=${AIRFLOW_UID} \
- UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
+ UPGRADE_INVALIDATION_STRING=${UPGRADE_INVALIDATION_STRING}
# Copy all scripts required for installation - changing any of those should
lead to
@@ -1573,7 +1572,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
bash /scripts/docker/create_prod_venv.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \
- ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
+ ${UPGRADE_INVALIDATION_STRING} == "" ]]; then \
bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
diff --git a/Dockerfile.ci b/Dockerfile.ci
index da275bbbff..fcd2a40068 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -706,7 +706,7 @@ function install_airflow() {
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra
'postgres' installations were therefore omitted.${COLOR_RESET}"
fi
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Remove airflow and all provider packages installed
before potentially${COLOR_RESET}"
echo
@@ -764,13 +764,12 @@ COPY <<"EOF" /install_additional_dependencies.sh
#!/usr/bin/env bash
set -euo pipefail
-: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}"
: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
function install_additional_dependencies() {
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Installing additional dependencies while upgrading
to newer dependencies${COLOR_RESET}"
echo
@@ -1324,11 +1323,11 @@ COPY --from=scripts install_airflow.sh /scripts/docker/
# dependency resolution and we do not need to limit the versions of the
dependencies
#
ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
-ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
+ARG UPGRADE_INVALIDATION_STRING=""
ARG VERSION_SUFFIX_FOR_PYPI=""
ENV
EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
- UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} \
+ UPGRADE_INVALIDATION_STRING=${UPGRADE_INVALIDATION_STRING} \
VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
# The goal of this line is to install the dependencies from the most current
pyproject.toml from sources
diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
index 253331e48b..19c58ebc2d 100644
--- a/dev/breeze/doc/ci/02_images.md
+++ b/dev/breeze/doc/ci/02_images.md
@@ -438,7 +438,7 @@ can be used for CI images:
| `AIRFLOW_CONSTRAINTS_LOCATION` |
| If not empty, it will override the source of
the constraints with the specified URL or file.
|
| `AIRFLOW_CONSTRAINTS_REFERENCE` |
| reference (branch or tag) from GitHub
repository from which constraints are used. By default it is set to
`constraints-main` but can be `constraints-2-X`. |
| `AIRFLOW_EXTRAS` | `all`
| extras to install
|
-| `UPGRADE_TO_NEWER_DEPENDENCIES` | `false`
| If set to a value different than "false" the
dependencies are upgraded to newer versions. In CI it is set to build id.
|
+| `UPGRADE_INVALIDATION_STRING` |
| If set to any random value the dependencies
are upgraded to newer versions. In CI it is set to build id.
|
| `AIRFLOW_PRE_CACHED_PIP_PACKAGES` | `true`
| Allows to pre-cache airflow PIP packages from
the GitHub of Apache Airflow This allows to optimize iterations for Image
builds and speeds up CI jobs. |
| `ADDITIONAL_AIRFLOW_EXTRAS` |
| additional extras to install
|
| `ADDITIONAL_PYTHON_DEPS` |
| additional Python dependencies to install
|
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index b93289627c..9199ae7344 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -309,7 +309,7 @@ am overview of possible labels and their meaning:
| all versions | all-versions, *-versions-* | Run tests
for all python and k8s versions.
|
| full tests needed | full-tests-needed | Run complete
set of tests (might be with default or all python/k8s versions)
|
| non committer build | is-committer-build | If set then
even for non-committer builds, the scripts used for images are used from target
branch. |
-| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set then
dependencies in the CI image build are upgraded to the newer ones.
|
+| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set to
true (default false) then dependencies in the CI image build are upgraded to
the newer ones. |
| use public runners | runs-on-as-json-default | Force using
public runners even for Committer runs.
|
diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
index 8f8182fd0a..7057212881 100644
--- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -73,7 +73,7 @@ class BuildCiParams(CommonBuildParams):
self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY",
self.constraints_github_repository)
self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image)
if self.upgrade_to_newer_dependencies:
- self._opt_arg("UPGRADE_TO_NEWER_DEPENDENCIES",
f"{random.randrange(2**32):x}")
+ self._opt_arg("UPGRADE_INVALIDATION_STRING",
f"{random.randrange(2**32):x}")
if self.eager_upgrade_additional_requirements:
# in case eager upgrade additional requirements have EOL,
connect them together
self._opt_arg(
diff --git a/docs/docker-stack/build-arg-ref.rst
b/docs/docker-stack/build-arg-ref.rst
index 7ec25173af..2d221a7c26 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-arg-ref.rst
@@ -130,66 +130,66 @@ the final image (RUNTIME) might not contain all the
dependencies that are needed
allowing to produce much more optimized images. See :ref:`Building optimized
images<image-build-optimized>`.
for examples of using those arguments.
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| Build argument | Default value
| Description |
-+==========================================+==========================================+==========================================+
-| ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false``
| If set to a value different than "false" |
-| |
| the dependencies are upgraded to newer |
-| |
| versions. In CI it is set to build id |
-| |
| to make sure subsequent builds are not |
-| |
| reusing cached images with same value. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_PYTHON_DEPS`` |
| Optional python packages to extend |
-| |
| the image with some extra dependencies. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``DEV_APT_COMMAND`` |
| Dev apt command executed before dev deps |
-| |
| are installed in the Build image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_DEV_APT_COMMAND`` |
| Additional Dev apt command executed |
-| |
| before dev dep are installed |
-| |
| in the Build image. Should start with |
-| |
| ``&&``. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``DEV_APT_DEPS`` | Empty - install default
dependencies | Dev APT dependencies installed |
-| | (see
``install_os_dependencies.sh``) | in the Build image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_DEV_APT_DEPS`` |
| Additional apt dev dependencies |
-| |
| installed in the Build image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_DEV_APT_ENV`` |
| Additional env variables defined |
-| |
| when installing dev deps. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``RUNTIME_APT_COMMAND`` |
| Runtime apt command executed before deps |
-| |
| are installed in the ``main`` stage. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_RUNTIME_APT_COMMAND`` |
| Additional Runtime apt command executed |
-| |
| before runtime dep are installed |
-| |
| in the ``main`` stage. Should start with |
-| |
| ``&&``. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``RUNTIME_APT_DEPS`` | Empty - install default
dependencies | Runtime APT dependencies installed |
-| | (see
``install_os_dependencies.sh``) | in the Main image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_RUNTIME_APT_DEPS`` |
| Additional apt runtime dependencies |
-| |
| installed in the Main image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``ADDITIONAL_RUNTIME_APT_ENV`` |
| Additional env variables defined |
-| |
| when installing runtime deps. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_MYSQL_CLIENT`` | ``true``
| Whether MySQL client should be installed |
-| |
| The mysql extra is removed from extras |
-| |
| if the client is not installed. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_MYSQL_CLIENT_TYPE`` | ``mariadb``
| Type of MySQL client library. This |
-| |
| can be ``mariadb`` or ``mysql`` |
-| |
| Regardless of the parameter, ``mariadb`` |
-| |
| will always be used on ARM. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_MSSQL_CLIENT`` | ``true``
| Whether MsSQL client should be installed |
-+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_POSTGRES_CLIENT`` | ``true``
| Whether Postgres client should be |
-| |
| installed |
-+------------------------------------------+------------------------------------------+------------------------------------------+
++------------------------------------+------------------------------------------+------------------------------------------+
+| Build argument | Default value
| Description |
++====================================+==========================================+==========================================+
+| ``UPGRADE_INVALIDATION_STRING`` |
| If set to a random, non-empty value |
+| |
| the dependencies are upgraded to newer |
+| |
| versions. In CI it is set to build id |
+| |
| to make sure subsequent builds are not |
+| |
| reusing cached images with same value. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_PYTHON_DEPS`` |
| Optional python packages to extend |
+| |
| the image with some extra dependencies. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``DEV_APT_COMMAND`` |
| Dev apt command executed before dev deps |
+| |
| are installed in the Build image. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_DEV_APT_COMMAND`` |
| Additional Dev apt command executed |
+| |
| before dev dep are installed |
+| |
| in the Build image. Should start with |
+| |
| ``&&``. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``DEV_APT_DEPS`` | Empty - install default dependencies
| Dev APT dependencies installed |
+| | (see ``install_os_dependencies.sh``)
| in the Build image. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_DEV_APT_DEPS`` |
| Additional apt dev dependencies |
+| |
| installed in the Build image. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_DEV_APT_ENV`` |
| Additional env variables defined |
+| |
| when installing dev deps. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``RUNTIME_APT_COMMAND`` |
| Runtime apt command executed before deps |
+| |
| are installed in the ``main`` stage. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_RUNTIME_APT_COMMAND`` |
| Additional Runtime apt command executed |
+| |
| before runtime dep are installed |
+| |
| in the ``main`` stage. Should start with |
+| |
| ``&&``. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``RUNTIME_APT_DEPS`` | Empty - install default dependencies
| Runtime APT dependencies installed |
+| | (see ``install_os_dependencies.sh``)
| in the Main image. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_RUNTIME_APT_DEPS`` |
| Additional apt runtime dependencies |
+| |
| installed in the Main image. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``ADDITIONAL_RUNTIME_APT_ENV`` |
| Additional env variables defined |
+| |
| when installing runtime deps. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_MYSQL_CLIENT`` | ``true``
| Whether MySQL client should be installed |
+| |
| The mysql extra is removed from extras |
+| |
| if the client is not installed. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_MYSQL_CLIENT_TYPE`` | ``mariadb``
| Type of MySQL client library. This |
+| |
| can be ``mariadb`` or ``mysql`` |
+| |
| Regardless of the parameter, ``mariadb`` |
+| |
| will always be used on ARM. |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_MSSQL_CLIENT`` | ``true``
| Whether MsSQL client should be installed |
++------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_POSTGRES_CLIENT`` | ``true``
| Whether Postgres client should be |
+| |
| installed |
++------------------------------------+------------------------------------------+------------------------------------------+
Installing Airflow using different methods
..........................................
diff --git a/scripts/ci/docker-compose/devcontainer.env
b/scripts/ci/docker-compose/devcontainer.env
index d07e9c629f..2b7cddb47e 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -71,7 +71,7 @@ START_AIRFLOW="false"
SUSPENDED_PROVIDERS_FOLDERS=""
TEST_TYPE=
UPGRADE_BOTO="false"
-UPGRADE_TO_NEWER_DEPENDENCIES="false"
+UPGRADE_INVALIDATION_STRING=""
VERBOSE="false"
VERBOSE_COMMANDS="false"
VERSION_SUFFIX_FOR_PYPI=
diff --git a/scripts/docker/install_additional_dependencies.sh
b/scripts/docker/install_additional_dependencies.sh
index 174d08956c..31ebf81217 100644
--- a/scripts/docker/install_additional_dependencies.sh
+++ b/scripts/docker/install_additional_dependencies.sh
@@ -18,7 +18,6 @@
# shellcheck shell=bash disable=SC2086
set -euo pipefail
-: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}"
: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
# shellcheck source=scripts/docker/common.sh
@@ -26,7 +25,7 @@ set -euo pipefail
# Installs additional dependencies passed as Argument to the Docker build
command
function install_additional_dependencies() {
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Installing additional dependencies while upgrading
to newer dependencies${COLOR_RESET}"
echo
diff --git a/scripts/docker/install_airflow.sh
b/scripts/docker/install_airflow.sh
index 63cdc701e2..dcaa4140da 100644
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -27,8 +27,8 @@
#
# AIRFLOW_VERSION_SPECIFICATION - optional specification for Airflow version
to install (
# might be ==2.0.2 for example or <3.0.0
-# UPGRADE_TO_NEWER_DEPENDENCIES - determines whether eager-upgrade should be
performed with the
-# dependencies (with
EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added)
+# UPGRADE_INVALIDATION_STRING - if set with random value determines whether
eager-upgrade should be done
+# for the dependencies (with
EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added)
#
# shellcheck shell=bash disable=SC2086
# shellcheck source=scripts/docker/common.sh
@@ -62,7 +62,7 @@ function install_airflow() {
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra
'postgres' installations were therefore omitted.${COLOR_RESET}"
fi
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
+ if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Remove airflow and all provider packages installed
before potentially${COLOR_RESET}"
echo
