This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 3a73110 Add extras when installing prod image from packages (#13432)
3a73110 is described below
commit 3a731108f530b6f7a90a9465faa0ae1daaa72552
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jan 2 13:37:29 2021 +0100
Add extras when installing prod image from packages (#13432)
In the latest change #13422 change in the way product images are
prepared removed extras from installed airflow - thus caused
failing production image verification check.
This change restores extras when airflow is installed from packages
---
Dockerfile | 36 ++++++++++++++++++++++++++++--------
Dockerfile.ci | 27 ++++++++++++++++++++-------
2 files changed, 48 insertions(+), 15 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 81737a1..da033cd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -265,20 +265,37 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- reinstalling_apache_airflow_packages=$(ls
/docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
- # We want to install apache airflow packages with constraints \
- if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
+ # We want to install apache airflow package with extras - that's why
it is a separate step \
+ # But we should still install them with all dependencies \
+ reinstalling_apache_airflow_package=$(ls
/docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
+ if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy
eager \
- --user ${reinstalling_apache_airflow_packages}; \
+ --user
"${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
+ # We want to install apache airflow package with constraints \
pip install --force-reinstall --upgrade --upgrade-strategy
only-if-needed \
- --user ${reinstalling_apache_airflow_packages}
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ --user
"${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]" --constraint
"${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
- # All the others we want to reinstall as-is, without dependencies \
+ reinstalling_apache_airflow_providers_packages=$(ls
/docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null ||
true); \
+ # We want to install apache airflow package with extras - that's why
it is a separate step \
+ # But we should still install them with all dependencies \
+ if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]];
then \
+ if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+ pip install --force-reinstall --upgrade --upgrade-strategy
eager \
+ --user ${reinstalling_apache_airflow_providers_packages}; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ else \
+ # We want to install apache airflow provider packages with
constraints \
+ pip install --force-reinstall --upgrade --upgrade-strategy
only-if-needed \
+ --user ${reinstalling_apache_airflow_providers_packages}
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ fi; \
+ fi ; \
+ # All the other packages we want to reinstall as-is, without
dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz}
2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
@@ -287,8 +304,11 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
- pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade
--upgrade-strategy eager; \
- pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ pip install --user ${ADDITIONAL_PYTHON_DEPS} \
+ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
+ --upgrade --upgrade-strategy eager; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}; \
else \
pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade
--upgrade-strategy only-if-needed \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 63360a0..7ac8760 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -337,26 +337,39 @@ COPY docker-context-files/ /docker-context-files/
# hadolint ignore=SC2086, SC2010
RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- reinstalling_apache_airflow_packages=$(ls
/docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
- # We want to install apache airflow packages with constraints \
- if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
+ reinstalling_apache_airflow_package=$(ls
/docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
+ # We want to install apache airflow package with constraints \
+ if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy
eager \
- --user ${reinstalling_apache_airflow_packages}; \
+ --user
${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
pip install --force-reinstall --upgrade --upgrade-strategy
only-if-needed \
- --user ${reinstalling_apache_airflow_packages}
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ --user
${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}] --constraint
"${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
- # All the others we want to reinstall as-is, without dependencies \
+ reinstalling_apache_airflow_providers_packages=$(ls
/docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null ||
true); \
+ # We want to install apache airflow provider packages with constraints
\
+ if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]];
then \
+ if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+ pip install --force-reinstall --upgrade --upgrade-strategy
eager \
+ --user ${reinstalling_apache_airflow_providers_packages}; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ else \
+ pip install --force-reinstall --upgrade --upgrade-strategy
only-if-needed \
+ --user ${reinstalling_apache_airflow_providers_packages}
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ fi; \
+ fi ; \
+ # All the other packages we want to reinstall as-is, without
dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz}
2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
pip install --force-reinstall --user --no-deps
${reinstalling_other_packages}; \
fi; \
- fi;
+ fi
# Copy all the www/ files we need to compile assets. Done as two separate COPY
# commands so as otherwise it copies the _contents_ of static/ in to www/
