This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ab5b2bf Workaround docker-compose-v2 env passing (#18887)
ab5b2bf is described below
commit ab5b2bfca6b5be57a19bdbfe2206319bd2e71ac9
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Oct 12 00:18:49 2021 +0200
Workaround docker-compose-v2 env passing (#18887)
* Workaround docker-compose-v2 env passing
Docker Compose v2 has environment parsing broken in many ways.
Until this is fixed, we cannot use env files, instead we must
set all the variables directly, because parsing variables without
values or parsing variables which have empty values is broken
in several ways. Some of the issues are closed but not released,
and until this is fixed, some extra code duplication and explicitly
setting all default variables to "" when needed should solve the
problem for both Docker-Compose v1 and Docker-Compose v2
---
breeze | 29 ++++++---------
scripts/ci/docker-compose/_docker_compose.env | 49 -------------------------
scripts/ci/docker-compose/base.yml | 52 +++++++++++++++++++++++++--
scripts/ci/libraries/_initialization.sh | 27 ++++++++++++++
scripts/in_container/entrypoint_ci.sh | 3 +-
5 files changed, 90 insertions(+), 70 deletions(-)
diff --git a/breeze b/breeze
index d6ef2cb..72bb8da 100755
--- a/breeze
+++ b/breeze
@@ -567,24 +567,6 @@ EOF
# Those are a convenience scripts that you might use to debug command
execution although
# In most cases they are used internally by Breeze.
#
-# Used Globals:
-# BRANCH_NAME
-# PYTHON_MAJOR_MINOR_VERSION
-# BACKEND
-# AIRFLOW_VERSION
-# INSTALL_AIRFLOW_VERSION
-# SSH_PORT
-# WEBSERVER_HOST_PORT
-# POSTGRES_HOST_PORT
-# POSTGRES_VERSION
-# MYSQL_HOST_PORT
-# MYSQL_VERSION
-# AIRFLOW_SOURCES
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_IMAGE_KUBERNETES
-# SQLITE_URL
-#
# Arguments:
#
# file to prepare
@@ -659,6 +641,17 @@ export SQLITE_URL="${SQLITE_URL}"
export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
export EXECUTOR="${EXECUTOR}"
+export START_AIRFLOW="${START_AIRFLOW}"
+export ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}"
+export ENABLED_SYSTEMS="${ENABLED_SYSTEMS}"
+export GITHUB_ACTIONS="${GITHUB_ACTIONS}"
+export ISSUE_ID="${ISSUE_ID}"
+export NUM_RUNS="${NUM_RUNS}"
+export RUN_TESTS="${RUN_TESTS}"
+export RUN_INTEGRATION_TESTS="${RUN_INTEGRATION_TESTS}"
+export RUN_SYSTEM_TESTS="${RUN_SYSTEM_TESTS}"
+export VERSION_SUFFIX_FOR_SVN="${VERSION_SUFFIX_FOR_SVN}"
+export VERSION_SUFFIX_FOR_PYPI="${VERSION_SUFFIX_FOR_PYPI}"
docker-compose ${command}
EOF
chmod u+x "${file}"
diff --git a/scripts/ci/docker-compose/_docker_compose.env
b/scripts/ci/docker-compose/_docker_compose.env
deleted file mode 100644
index d206af4..0000000
--- a/scripts/ci/docker-compose/_docker_compose.env
+++ /dev/null
@@ -1,49 +0,0 @@
-AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
-AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}"
-BACKEND="${BACKEND}"
-BREEZE="${BREEZE}"
-CI="${CI}"
-CI_BUILD_ID="${CI_BUILD_ID}"
-CI_JOB_ID="${CI_JOB_ID}"
-CI_EVENT_TYPE="${CI_EVENT_TYPE}"
-CI_TARGET_REPO="${CI_TARGET_REPO}"
-CI_TARGET_BRANCH="${CI_TARGET_BRANCH}"
-COMMIT_SHA="${COMMIT_SHA}"
-DB_RESET="${DB_RESET}"
-DEFAULT_BRANCH="${DEFAULT_BRANCH}"
-DEFAULT_CONSTRAINTS_BRANCH="${DEFAULT_CONSTRAINTS_BRANCH}"
-ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}"
-ENABLED_SYSTEMS="${ENABLED_SYSTEMS}"
-GITHUB_ACTIONS="${GITHUB_ACTIONS}"
-GITHUB_REGISTRY_PULL_IMAGE_TAG="${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
-HOST_USER_ID="${HOST_USER_ID}"
-HOST_GROUP_ID="${HOST_GROUP_ID}"
-HOST_OS="${HOST_OS}"
-HOST_HOME="${HOST_HOME}"
-INIT_SCRIPT_FILE="${INIT_SCRIPT_FILE}"
-INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
-GENERATE_CONSTRAINTS_MODE="${GENERATE_CONSTRAINTS_MODE}"
-INSTALL_PROVIDERS_FROM_SOURCES="${INSTALL_PROVIDERS_FROM_SOURCES}"
-USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
-USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
-ISSUE_ID="${ISSUE_ID}"
-LOAD_DEFAULT_CONNECTIONS="${LOAD_DEFAULT_CONNECTIONS}"
-LOAD_EXAMPLES="${LOAD_EXAMPLES}"
-MYSQL_VERSION="${MYSQL_VERSION}"
-NUM_RUNS="${NUM_RUNS}"
-PACKAGE_FORMAT="${PACKAGE_FORMAT}"
-POSTGRES_VERSION="${POSTGRES_VERSION}"
-PRINT_INFO_FROM_SCRIPTS="${PRINT_INFO_FROM_SCRIPTS}"
-PYTHONDONTWRITEBYTECODE="${PYTHONDONTWRITEBYTECODE}"
-PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}"
-RUN_TESTS="${RUN_TESTS}"
-RUN_INTEGRATION_TESTS="${RUN_INTEGRATION_TESTS}"
-RUN_SYSTEM_TESTS="${RUN_SYSTEM_TESTS}"
-START_AIRFLOW="${START_AIRFLOW}"
-TEST_TYPE="${TEST_TYPE}"
-UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}"
-VERBOSE="${VERBOSE}"
-VERBOSE_COMMANDS="${VERBOSE_COMMANDS}"
-VERSION_SUFFIX_FOR_PYPI="${VERSION_SUFFIX_FOR_PYPI}"
-VERSION_SUFFIX_FOR_SVN="${VERSION_SUFFIX_FOR_SVN}"
-WHEEL_VERSION="${WHEEL_VERSION}"
diff --git a/scripts/ci/docker-compose/base.yml
b/scripts/ci/docker-compose/base.yml
index c9fed6a..4ecf6ee 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -26,8 +26,56 @@ services:
-
CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0
- KUBECONFIG=/files/.kube/config
- HOST_HOME=${HOME}
- env_file:
- - _docker_compose.env
+ # We need all those env variables here because docker-compose-v2 does
not really work well
+ # With env files and there are many problems with it:
+ - AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE}
+ - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+ - BACKEND=${BACKEND}
+ - BREEZE=${BREEZE}
+ - CI=${CI}
+ - CI_BUILD_ID=${CI_BUILD_ID}
+ - CI_JOB_ID=${CI_JOB_ID}
+ - CI_EVENT_TYPE=${CI_EVENT_TYPE}
+ - CI_TARGET_REPO=${CI_TARGET_REPO}
+ - CI_TARGET_BRANCH=${CI_TARGET_BRANCH}
+ - COMMIT_SHA=${COMMIT_SHA}
+ - DB_RESET=${DB_RESET}
+ - DEFAULT_BRANCH=${DEFAULT_BRANCH}
+ - DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}
+ - ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS}
+ - ENABLED_SYSTEMS=${ENABLED_SYSTEMS}
+ - GITHUB_ACTIONS=${GITHUB_ACTIONS}
+ - GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG}
+ - HOST_USER_ID=${HOST_USER_ID}
+ - HOST_GROUP_ID=${HOST_GROUP_ID}
+ - HOST_OS=${HOST_OS}
+ - INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE}
+ - INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION}
+ - GENERATE_CONSTRAINTS_MODE=${GENERATE_CONSTRAINTS_MODE}
+ - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES}
+ - USE_AIRFLOW_VERSION=${USE_AIRFLOW_VERSION}
+ - USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST}
+ - ISSUE_ID=${ISSUE_ID}
+ - LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
+ - LOAD_EXAMPLES=${LOAD_EXAMPLES}
+ - MYSQL_VERSION=${MYSQL_VERSION}
+ - NUM_RUNS=${NUM_RUNS}
+ - PACKAGE_FORMAT=${PACKAGE_FORMAT}
+ - POSTGRES_VERSION=${POSTGRES_VERSION}
+ - PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS}
+ - PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE}
+ - PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+ - RUN_TESTS=${RUN_TESTS}
+ - RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS}
+ - RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS}
+ - START_AIRFLOW=${START_AIRFLOW}
+ - TEST_TYPE=${TEST_TYPE}
+ - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
+ - VERBOSE=${VERBOSE}
+ - VERBOSE_COMMANDS=${VERBOSE_COMMANDS}
+ - VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
+ - VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN}
+ - WHEEL_VERSION=${WHEEL_VERSION}
volumes:
# Pass docker to inside of the container so that Kind and Moto tests can
use it.
- /var/run/docker.sock:/var/run/docker.sock
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index ff9fb29..31e17e4 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -311,6 +311,19 @@ function initialization::initialize_force_variables() {
# Can be set to true to skip if the image is newer in registry
export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"}
+
+ # integrations are disabled by default
+ export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
+
+ # systems are disabled by default
+ export ENABLED_SYSTEMS=${ENABLED_SYSTEMS:=""}
+
+ # no issue id by default (quarantined builds only)
+ export ISSUE_ID=${ISSUE_ID:=""}
+
+ # no NUM_RUNS by default (quarantined builds only)
+ export NUM_RUNS=${NUM_RUNS:=""}
+
}
# Determine information about the host
@@ -576,10 +589,24 @@ function initialization::initialize_test_variables() {
# In case we want to force certain test type to run, this variable should
be set to this type
# Otherwise TEST_TYPEs to run will be derived from TEST_TYPES
space-separated string
export FORCE_TEST_TYPE=${FORCE_TEST_TYPE:=""}
+
+ # Do not run tests by default
+ export RUN_TESTS=${RUN_TESTS:="false"}
+
+ # Do not run integration tests by default
+ export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:="false"}
+
+ # Do not run system tests by default
+ export RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS:="false"}
+
}
function initialization::initialize_package_variables() {
+ # default package format
export PACKAGE_FORMAT=${PACKAGE_FORMAT:="wheel"}
+ # default version suffixes
+ export VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI:=""}
+ export VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN:=""}
}
diff --git a/scripts/in_container/entrypoint_ci.sh
b/scripts/in_container/entrypoint_ci.sh
index 29f5210..0e09c8a 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -52,7 +52,7 @@ else
export RUN_AIRFLOW_1_10="false"
fi
-if [[ -z ${USE_AIRFLOW_VERSION=} ]]; then
+if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then
export PYTHONPATH=${AIRFLOW_SOURCES}
echo
echo "Using already installed airflow version"
@@ -184,6 +184,7 @@ ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
cd "${AIRFLOW_SOURCES}"
+echo "START_AIRFLOW:=${START_AIRFLOW}"
if [[ ${START_AIRFLOW:="false"} == "true" ]]; then
export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
