This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bec006e Fix constraint generation properly (#17964)
bec006e is described below
commit bec006e418706f11d92271d0b9f2a46afccc80e2
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Sep 2 11:24:16 2021 +0200
Fix constraint generation properly (#17964)
The #17939 did not fix the problem finally. It turned out that
one more change was needed - since we now always upgrade to latest
dependencies in `push` and `schedule` type of build we do not need
to check for the variable UPGRADE_TO_NEWER_DEPENDENCIES (which
was not set in "Build Image" step.
This fixes it, but also changes the constraint generation to add
comments in the generated constraint files, describing how and
why the files are generated.
---
CI.rst | 8 ++--
scripts/ci/selective_ci_checks.sh | 21 +++++------
scripts/in_container/run_generate_constraints.sh | 48 +++++++++++++++++++++---
3 files changed, 56 insertions(+), 21 deletions(-)
diff --git a/CI.rst b/CI.rst
index 61b0d50..6cc97cc 100644
--- a/CI.rst
+++ b/CI.rst
@@ -124,7 +124,7 @@ You can use those variables when you try to reproduce the
build locally.
| | | |
| case of Postgres or MySQL. However, |
| | | |
| it requires to perform manual init/reset |
| | | |
| if you stop the environment. |
-+-----------------------------------------+-------------+--------------+-------------+-------------------------------------------------+
++-----------------------------------------+-------------+--------------+-------------+------------------------------------------------+
| Mount variables
|
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| ``MOUNT_SELECTED_LOCAL_SOURCES`` | true | false |
false | Determines whether local sources are |
@@ -209,7 +209,7 @@ You can use those variables when you try to reproduce the
build locally.
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| Git variables
|
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
-| COMMIT_SHA | | GITHUB_SHA |
GITHUB_SHA | SHA of the commit of the build is run |
+| ``COMMIT_SHA`` | | GITHUB_SHA |
GITHUB_SHA | SHA of the commit of the build is run |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| Verbosity variables
|
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
@@ -255,7 +255,7 @@ You can use those variables when you try to reproduce the
build locally.
| | | |
| of the airflow repository |
| | | |
| ("constraints-main, "constraints-2-0") |
| | | |
| but when this flag is set to anything but false |
-| | | |
| (for example commit SHA), they are not used |
+| | | |
| (for example random value), they are not used |
| | | |
| used and "eager" upgrade strategy is used |
| | | |
| when installing dependencies. We set it |
| | | |
| to true in case of direct pushes (merges) |
@@ -265,7 +265,7 @@ You can use those variables when you try to reproduce the
build locally.
| | | |
| we automatically push latest set of |
| | | |
| "tested" constraints to the repository. |
| | | |
| |
-| | | |
| Setting the value to commit SHA is best way |
+| | | |
| Setting the value to random value is best way |
| | | |
| to assure that constraints are upgraded even if |
| | | |
| there is no change to setup.py |
| | | |
| |
diff --git a/scripts/ci/selective_ci_checks.sh
b/scripts/ci/selective_ci_checks.sh
index 15020ad..10aa128 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -43,16 +43,14 @@ else
fi
function check_upgrade_to_newer_dependencies_needed() {
- # shellcheck disable=SC2153
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ||
- ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} ==
"scheduled" ]]; then
- # Trigger upgrading to latest constraints when we are in push or
schedule event or when it is forced
- # By UPGRADE_TO_NEWER_DEPENDENCIES set to non-false. The variable is
set to
- # SHA of the merge commit - so that it always triggers rebuilding
layer in the docker image
+ if [[ ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} ==
"scheduled" ]]; then
+ # Trigger upgrading to latest constraints when we are in push or
schedule event. We use the
+ # random string so that it always triggers rebuilding layer in the
docker image
# Each build that upgrades to latest constraints will get truly latest
constraints, not those
- # Cached in the image if we set it to "true". This
upgrade_to_newer_dependencies variable
- # can later be overridden in case we find that setup.* files changed
(see below)
- upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}"
+ # cached in the image because docker layer will get invalidated.
+ # This upgrade_to_newer_dependencies variable can later be overridden
+ # in case we find that any of the setup.* files changed (see below)
+ upgrade_to_newer_dependencies="${RANDOM}"
fi
}
@@ -353,8 +351,9 @@ function check_if_setup_files_changed() {
if [[ $(count_changed_files) != "0" ]]; then
# In case the setup files changed, we automatically force upgrading to
newer dependencies
- # no matter what was set before
- upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}"
+ # no matter what was set before. We set it to random number to make
sure that it will be
+ # always invalidating the layer in Docker that triggers installing the
dependencies
+ upgrade_to_newer_dependencies="${RANDOM}"
fi
start_end::group_end
}
diff --git a/scripts/in_container/run_generate_constraints.sh
b/scripts/in_container/run_generate_constraints.sh
index d71fc9d..402ac85 100755
--- a/scripts/in_container/run_generate_constraints.sh
+++ b/scripts/in_container/run_generate_constraints.sh
@@ -20,6 +20,12 @@
CONSTRAINTS_DIR="/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}"
+LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt"
+CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt"
+
+mkdir -pv "${CONSTRAINTS_DIR}"
+
+
if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]]; then
AIRFLOW_CONSTRAINTS="constraints-no-providers"
NO_PROVIDERS_EXTRAS=$(python -c 'import setup;
print(",".join(setup.CORE_EXTRAS_REQUIREMENTS.keys()))')
@@ -31,11 +37,34 @@ if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]];
then
echo "Install airflow with [${NO_PROVIDERS_EXTRAS}] extras only (uninstall
all packages first)."
echo
install_local_airflow_with_eager_upgrade "[${NO_PROVIDERS_EXTRAS}]"
+ cat <<EOF >"${CURRENT_CONSTRAINT_FILE}"
+#
+# This constraints file was automatically generated on $(date -u
+'%Y-%m-%dT%H:%M:%SZ')
+# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of
Airflow.
+# This variant of constraints install just the 'bare' 'apache-airflow' package
build from the HEAD of
+# the branch, without installing any of the providers.
+#
+# Those constraints represent the "newest" dependencies airflow could use, if
providers did not limit
+# Airflow in any way.
+#
+EOF
elif [[ ${GENERATE_CONSTRAINTS_MODE} == "source-providers" ]]; then
AIRFLOW_CONSTRAINTS="constraints-source-providers"
echo
echo "Providers are already installed from sources."
echo
+ cat <<EOF >"${CURRENT_CONSTRAINT_FILE}"
+#
+# This constraints file was automatically generated on $(date -u
+'%Y-%m-%dT%H:%M:%SZ')
+# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of
Airflow.
+# This variant of constraints install uses the HEAD of the branch version of
both
+# 'apache-airflow' package nd all available community provider packages.
+#
+# Those constraints represent the dependencies that are used by all pull
requests when they are build in CI.
+# They represent "latest" and greatest set of constraints that HEAD of the
"apache-airflow" package should
+# Install with "HEAD" of providers. Those are the only constraints that are
used by our CI builds.
+#
+EOF
elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]]; then
AIRFLOW_CONSTRAINTS="constraints"
echo
@@ -43,6 +72,18 @@ elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]];
then
echo
install_all_providers_from_pypi_with_eager_upgrade
else
+ cat <<EOF >"${CURRENT_CONSTRAINT_FILE}"
+#
+# This constraints file was automatically generated on $(date -u
+'%Y-%m-%dT%H:%M:%SZ')
+# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of
Airflow.
+# This variant of constraints install uses the HEAD of the branch version for
'apache-airflow' but installs
+# the providers from PIP-released packages at the moment of the constraint
generation.
+#
+# Those constraints are actually those that that regular users use to install
released version of Airflow.
+# We also use those constraints after "apache-airflow" is released and the
constraints are tagged with
+# "constraints-X.Y.Z" tag to build the production image for that version.
+#
+EOF
echo
echo "${COLOR_RED}Error! GENERATE_CONSTRAINTS_MODE has wrong value:
'${GENERATE_CONSTRAINTS_MODE}' ${COLOR_RESET}"
echo
@@ -51,11 +92,6 @@ fi
readonly AIRFLOW_CONSTRAINTS
-LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt"
-CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt"
-
-mkdir -pv "${CONSTRAINTS_DIR}"
-
CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt";
readonly CONSTRAINTS_LOCATION
@@ -69,7 +105,7 @@ echo
pip freeze | sort | \
grep -v "apache_airflow" | \
grep -v "@" | \
- grep -v "/opt/airflow" >"${CURRENT_CONSTRAINT_FILE}"
+ grep -v "/opt/airflow" >>"${CURRENT_CONSTRAINT_FILE}"
echo
echo "Constraints generated in ${CURRENT_CONSTRAINT_FILE}"
