[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402413510
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,333 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG WWW_FOLDER="www"
+
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+
+ARG PYTHON_BASE_IMAGE
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Install curl and gnupg2 - needed to download nodejs in the next step
+RUN apt-get update \
+&& apt-get install -y --no-install-recommends \
+   curl \
+   gnupg2 \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install basic apt dependencies
+RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
+&& curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > 
/dev/null \
+&& echo "deb https://dl.yarnpkg.com/debian/ stable main" > 
/etc/apt/sources.list.d/yarn.list \
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+&& mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   nodejs \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402413029
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS BETA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG 
AIRFLOW_EXTRAS="async,azure_blob_storage,azure_cosmos,azure_container_instances,celery,crypto,elasticsearch,gcp,kubernetes,mysql,postgres,s3,emr,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_USER="airflow"
+ARG AIRFLOW_GROUP="airflow"
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG AIRFLOW_EXTRAS
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo "Airflow org: ${AIRFLOW_ORG}"; \
+echo "Airflow repo: ${AIRFLOW_REPO}"; \
+echo "Airflow repo url: ${AIRFLOW_REPO_URL}"; \
+echo "Airflow extras: ${AIRFLOW_EXTRAS}" ;\
+echo "PIP version: ${PIP_VERSION}" ;\
+echo "Cassandra concurrency: ${CASS_DRIVER_BUILD_CONCURRENCY}" ;\
+echo
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+# Install basic apt dependencies
+RUN mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   sasl2-bin \
+   software-properties-common \
+   sqlite3 \
+   sudo \
+   unixodbc \
+   unixodbc-dev \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
+&& GNUPGHOME="$(mktemp -d)" \
+&& export GNUPGHOME \
+&& for KEYSERVER in $(shuf -e 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402328081
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS BETA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG 
AIRFLOW_EXTRAS="async,azure_blob_storage,azure_cosmos,azure_container_instances,celery,crypto,elasticsearch,gcp,kubernetes,mysql,postgres,s3,emr,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_USER="airflow"
+ARG AIRFLOW_GROUP="airflow"
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG AIRFLOW_EXTRAS
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo "Airflow org: ${AIRFLOW_ORG}"; \
+echo "Airflow repo: ${AIRFLOW_REPO}"; \
+echo "Airflow repo url: ${AIRFLOW_REPO_URL}"; \
+echo "Airflow extras: ${AIRFLOW_EXTRAS}" ;\
+echo "PIP version: ${PIP_VERSION}" ;\
+echo "Cassandra concurrency: ${CASS_DRIVER_BUILD_CONCURRENCY}" ;\
+echo
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+# Install basic apt dependencies
+RUN mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   sasl2-bin \
+   software-properties-common \
+   sqlite3 \
+   sudo \
+   unixodbc \
+   unixodbc-dev \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
+&& GNUPGHOME="$(mktemp -d)" \
+&& export GNUPGHOME \
+&& for KEYSERVER in $(shuf -e 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402324811
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,333 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG WWW_FOLDER="www"
+
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+
+ARG PYTHON_BASE_IMAGE
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Install curl and gnupg2 - needed to download nodejs in the next step
+RUN apt-get update \
+&& apt-get install -y --no-install-recommends \
+   curl \
+   gnupg2 \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install basic apt dependencies
+RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
+&& curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > 
/dev/null \
+&& echo "deb https://dl.yarnpkg.com/debian/ stable main" > 
/etc/apt/sources.list.d/yarn.list \
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+&& mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   nodejs \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402323779
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,333 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG WWW_FOLDER="www"
+
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+
+ARG PYTHON_BASE_IMAGE
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Install curl and gnupg2 - needed to download nodejs in the next step
+RUN apt-get update \
+&& apt-get install -y --no-install-recommends \
+   curl \
+   gnupg2 \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install basic apt dependencies
+RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
+&& curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > 
/dev/null \
+&& echo "deb https://dl.yarnpkg.com/debian/ stable main" > 
/etc/apt/sources.list.d/yarn.list \
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+&& mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   nodejs \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402322231
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS BETA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG 
AIRFLOW_EXTRAS="async,azure_blob_storage,azure_cosmos,azure_container_instances,celery,crypto,elasticsearch,gcp,kubernetes,mysql,postgres,s3,emr,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_USER="airflow"
+ARG AIRFLOW_GROUP="airflow"
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG AIRFLOW_EXTRAS
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo "Airflow org: ${AIRFLOW_ORG}"; \
+echo "Airflow repo: ${AIRFLOW_REPO}"; \
+echo "Airflow repo url: ${AIRFLOW_REPO_URL}"; \
+echo "Airflow extras: ${AIRFLOW_EXTRAS}" ;\
+echo "PIP version: ${PIP_VERSION}" ;\
+echo "Cassandra concurrency: ${CASS_DRIVER_BUILD_CONCURRENCY}" ;\
+echo
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+# Install basic apt dependencies
+RUN mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   sasl2-bin \
+   software-properties-common \
+   sqlite3 \
+   sudo \
+   unixodbc \
+   unixodbc-dev \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
+&& GNUPGHOME="$(mktemp -d)" \
+&& export GNUPGHOME \
+&& for KEYSERVER in $(shuf -e 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r402321490
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS BETA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG 
AIRFLOW_EXTRAS="async,azure_blob_storage,azure_cosmos,azure_container_instances,celery,crypto,elasticsearch,gcp,kubernetes,mysql,postgres,s3,emr,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_USER="airflow"
+ARG AIRFLOW_GROUP="airflow"
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG AIRFLOW_EXTRAS
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo "Airflow org: ${AIRFLOW_ORG}"; \
+echo "Airflow repo: ${AIRFLOW_REPO}"; \
+echo "Airflow repo url: ${AIRFLOW_REPO_URL}"; \
+echo "Airflow extras: ${AIRFLOW_EXTRAS}" ;\
+echo "PIP version: ${PIP_VERSION}" ;\
+echo "Cassandra concurrency: ${CASS_DRIVER_BUILD_CONCURRENCY}" ;\
+echo
+
+# Make sure noninteractive debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 
LC_ALL=C.UTF-8 \
+LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+# Note missing man directories on debian-buster
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+# Install basic apt dependencies
+RUN mkdir -pv /usr/share/man/man1 \
+&& mkdir -pv /usr/share/man/man7 \
+&& apt-get update \
+&& apt-get install -y --no-install-recommends \
+   apt-transport-https \
+   apt-utils \
+   build-essential \
+   ca-certificates \
+   curl \
+   gnupg \
+   dirmngr \
+   freetds-bin \
+   freetds-dev \
+   gosu \
+   krb5-user \
+   ldap-utils \
+   libffi-dev \
+   libkrb5-dev \
+   libpq-dev \
+   libsasl2-2 \
+   libsasl2-dev \
+   libsasl2-modules \
+   libssl-dev \
+   locales  \
+   lsb-release \
+   openssh-client \
+   postgresql-client \
+   python-selinux \
+   sasl2-bin \
+   software-properties-common \
+   sqlite3 \
+   sudo \
+   unixodbc \
+   unixodbc-dev \
+&& apt-get autoremove -yqq --purge \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# Install MySQL client from Oracle repositories (Debian installs mariadb)
+RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
+&& GNUPGHOME="$(mktemp -d)" \
+&& export GNUPGHOME \
+&& for KEYSERVER in $(shuf -e 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401530747
 
 

 ##
 File path: entrypoint.sh
 ##
 @@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Might be empty
+AIRFLOW_COMMAND="${1}"
+
+set -euo pipefail
+
+function verify_db_connection {
+DB_URL="${1}"
+
+DB_CHECK_MAX_COUNT=${MAX_DB_CHECK_COUNT:=10}
+DB_CHECK_SLEEP_TIME=${DB_CHECK_SLEEP_TIME:=2}
+
+export DETECTED_DB_BACKEND=""
+export DETECTED_DB_USER=""
+export DETECTED_DB_HOST=""
+export DETECTED_DB_PORT=""
+export DETECTED_DB_SCHEMA=""
+export DETECTED_DB_PARAMS=""
 
 Review comment:
   (It doesn't matter/affect anything much, and wouldn't have commented unless 
there were other changes to make.)


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401511888
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
 
 Review comment:
   Oh `docker pull apache-airflow` would be fine, yeah.
   
   How quickly can we iterate on our Dockerfile if needed once it makes it in 
to the "library"


This is an automated message from the 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401208609
 
 

 ##
 File path: entrypoint.sh
 ##
 @@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Might be empty
+AIRFLOW_COMMAND="${1}"
+
+set -euo pipefail
+
+function verify_db_connection {
+DB_URL="${1}"
+
+DB_CHECK_MAX_COUNT=${MAX_DB_CHECK_COUNT:=10}
+DB_CHECK_SLEEP_TIME=${DB_CHECK_SLEEP_TIME:=2}
+
+export DETECTED_DB_BACKEND=""
+export DETECTED_DB_USER=""
+export DETECTED_DB_HOST=""
+export DETECTED_DB_PORT=""
+export DETECTED_DB_SCHEMA=""
+export DETECTED_DB_PARAMS=""
+
+if [[ ${DB_URL} != sqlite* ]]; then
+# Auto-detect DB parameters
+[[ ${DB_URL} =~ ([^:]*)://([^@/]*)@?([^/:]*):?([0-9]*)/([^\?]*)\??(.*) 
]] && \
+DETECTED_DB_BACKEND=${BASH_REMATCH[1]} &&
+DETECTED_DB_USER=${BASH_REMATCH[2]} &&
+DETECTED_DB_HOST=${BASH_REMATCH[3]} &&
+DETECTED_DB_PORT=${BASH_REMATCH[4]} &&
+DETECTED_DB_SCHEMA=${BASH_REMATCH[5]} &&
+DETECTED_DB_PARAMS=${BASH_REMATCH[6]}
+
+echo DB_BACKEND="${DB_BACKEND:=${DETECTED_DB_BACKEND}}"
+
+if [[ -z "${DETECTED_DB_PORT}" ]]; then
+if [[ ${DB_BACKEND} == "postgres"* ]]; then
+DETECTED_DB_PORT=5432
+elif [[ ${DB_BACKEND} == "mysql"* ]]; then
+DETECTED_DB_PORT=3306
+fi
+fi
+
+DETECTED_DB_HOST=${DETECTED_DB_HOST:="localhost"}
+
+# Allow the DB parameters to be overridden by environment variable
+echo DB_HOST="${DB_HOST:=${DETECTED_DB_HOST}}"
+echo DB_PORT="${DB_PORT:=${DETECTED_DB_PORT}}"
+
+while true
+do
+set +e
+LAST_CHECK_RESULT=$(nc -zvv "${DB_HOST}" "${DB_PORT}" >/dev/null 
2>&1)
+RES=$?
+set -e
+if [[ ${RES} == 0 ]]; then
+echo
+break
+else
+echo -n "."
+DB_CHECK_MAX_COUNT=$((DB_CHECK_MAX_COUNT-1))
+fi
+if [[ ${DB_CHECK_MAX_COUNT} == 0 ]]; then
+echo
+echo "ERROR! Maximum number of retries (${DB_CHECK_MAX_COUNT}) 
reached while checking ${DB_BACKEND} db. Exiting"
+echo
+break
+else
+sleep "${DB_CHECK_SLEEP_TIME}"
+fi
+done
+if [[ ${RES} != 0 ]]; then
+echo "ERROR: ${BACKEND} db could not be reached!"
+echo
+echo "${LAST_CHECK_RESULT}"
+echo
+export EXIT_CODE=${RES}
+fi
+fi
+}
+
+# if no DB configured - use sqlite db by default
+AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__CORE__SQL_ALCHEMY_CONN:="sqlite:///${AIRFLOW_HOME}/airflow.db"}"
+
+verify_db_connection "${AIRFLOW__CORE__SQL_ALCHEMY_CONN}"
+
+AIRFLOW__CELERY__BROKER_URL=${AIRFLOW__CELERY__BROKER_URL:=}
+
+if [[ -n ${AIRFLOW__CELERY__BROKER_URL} ]] && \
+[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|flower)$ ]]; then
+verify_db_connection "${AIRFLOW__CELERY__BROKER_URL}"
+fi
+
+if [[ ${AIRFLOW__CORE__SQL_ALCHEMY_CONN} == sqlite* ]] && \
+[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|webserver)$ ]] || \
+[[ ${DB_RESET:="false"} == "true" ]]; then
+if [[ ${AIRFLOW_VERSION} == *1.10* ]]; then
 
 Review comment:
   This could match 2.1.10 in future


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401208246
 
 

 ##
 File path: entrypoint.sh
 ##
 @@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Might be empty
+AIRFLOW_COMMAND="${1}"
+
+set -euo pipefail
+
+function verify_db_connection {
+DB_URL="${1}"
+
+DB_CHECK_MAX_COUNT=${MAX_DB_CHECK_COUNT:=10}
+DB_CHECK_SLEEP_TIME=${DB_CHECK_SLEEP_TIME:=2}
+
+export DETECTED_DB_BACKEND=""
+export DETECTED_DB_USER=""
+export DETECTED_DB_HOST=""
+export DETECTED_DB_PORT=""
+export DETECTED_DB_SCHEMA=""
+export DETECTED_DB_PARAMS=""
+
+if [[ ${DB_URL} != sqlite* ]]; then
+# Auto-detect DB parameters
+[[ ${DB_URL} =~ ([^:]*)://([^@/]*)@?([^/:]*):?([0-9]*)/([^\?]*)\??(.*) 
]] && \
+DETECTED_DB_BACKEND=${BASH_REMATCH[1]} &&
+DETECTED_DB_USER=${BASH_REMATCH[2]} &&
+DETECTED_DB_HOST=${BASH_REMATCH[3]} &&
+DETECTED_DB_PORT=${BASH_REMATCH[4]} &&
+DETECTED_DB_SCHEMA=${BASH_REMATCH[5]} &&
+DETECTED_DB_PARAMS=${BASH_REMATCH[6]}
+
+echo DB_BACKEND="${DB_BACKEND:=${DETECTED_DB_BACKEND}}"
+
+if [[ -z "${DETECTED_DB_PORT}" ]]; then
+if [[ ${DB_BACKEND} == "postgres"* ]]; then
+DETECTED_DB_PORT=5432
+elif [[ ${DB_BACKEND} == "mysql"* ]]; then
+DETECTED_DB_PORT=3306
+fi
+fi
+
+DETECTED_DB_HOST=${DETECTED_DB_HOST:="localhost"}
+
+# Allow the DB parameters to be overridden by environment variable
+echo DB_HOST="${DB_HOST:=${DETECTED_DB_HOST}}"
+echo DB_PORT="${DB_PORT:=${DETECTED_DB_PORT}}"
+
+while true
+do
+set +e
+LAST_CHECK_RESULT=$(nc -zvv "${DB_HOST}" "${DB_PORT}" >/dev/null 
2>&1)
+RES=$?
+set -e
+if [[ ${RES} == 0 ]]; then
+echo
+break
+else
+echo -n "."
+DB_CHECK_MAX_COUNT=$((DB_CHECK_MAX_COUNT-1))
+fi
+if [[ ${DB_CHECK_MAX_COUNT} == 0 ]]; then
+echo
+echo "ERROR! Maximum number of retries (${DB_CHECK_MAX_COUNT}) 
reached while checking ${DB_BACKEND} db. Exiting"
+echo
+break
+else
+sleep "${DB_CHECK_SLEEP_TIME}"
+fi
+done
+if [[ ${RES} != 0 ]]; then
+echo "ERROR: ${BACKEND} db could not be reached!"
+echo
+echo "${LAST_CHECK_RESULT}"
+echo
+export EXIT_CODE=${RES}
+fi
+fi
+}
+
+# if no DB configured - use sqlite db by default
+AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__CORE__SQL_ALCHEMY_CONN:="sqlite:///${AIRFLOW_HOME}/airflow.db"}"
+
+verify_db_connection "${AIRFLOW__CORE__SQL_ALCHEMY_CONN}"
+
+AIRFLOW__CELERY__BROKER_URL=${AIRFLOW__CELERY__BROKER_URL:=}
+
+if [[ -n ${AIRFLOW__CELERY__BROKER_URL} ]] && \
+[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|flower)$ ]]; then
+verify_db_connection "${AIRFLOW__CELERY__BROKER_URL}"
+fi
+
+if [[ ${AIRFLOW__CORE__SQL_ALCHEMY_CONN} == sqlite* ]] && \
+[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|webserver)$ ]] || \
+[[ ${DB_RESET:="false"} == "true" ]]; then
+if [[ ${AIRFLOW_VERSION} == *1.10* ]]; then
+airflow reset_db -y >/dev/null 2>&1
 
 Review comment:
   Woah. This shouldn't even be an option in a production image, it seems 
wildly dangerous.


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401206953
 
 

 ##
 File path: entrypoint.sh
 ##
 @@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Might be empty
+AIRFLOW_COMMAND="${1}"
+
+set -euo pipefail
+
+function verify_db_connection {
+DB_URL="${1}"
+
+DB_CHECK_MAX_COUNT=${MAX_DB_CHECK_COUNT:=10}
+DB_CHECK_SLEEP_TIME=${DB_CHECK_SLEEP_TIME:=2}
+
+export DETECTED_DB_BACKEND=""
+export DETECTED_DB_USER=""
+export DETECTED_DB_HOST=""
+export DETECTED_DB_PORT=""
+export DETECTED_DB_SCHEMA=""
+export DETECTED_DB_PARAMS=""
 
 Review comment:
   ```suggestion
   DETECTED_DB_BACKEND=""
   DETECTED_DB_USER=""
   DETECTED_DB_HOST=""
   DETECTED_DB_PORT=""
   DETECTED_DB_SCHEMA=""
   DETECTED_DB_PARAMS=""
   ```
   
   These aren't used by any subcommands, so no need to export them.


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401205424
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,373 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401204155
 
 

 ##
 File path: IMAGES.rst
 ##
 @@ -0,0 +1,389 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+.. contents:: :local:
+
+Airflow docker images
+=
+
+Airflow has two images (build from Dockerfiles):
+
+* CI image (Dockerfile.ci) - used for running tests and local development
+* Production image (Dockerfile) - used to run production-ready Airflow 
installations
+
+Image naming conventions
+
+
+The images are named as follows:
+
+``apache/airflow:-python[-ci][-manifest]``
+
+where:
+
+* BRANCH_OR_TAG - branch or tag used when creating the image. Examples: 
master, v1-10-test, 1.10.10
+  The ``master`` and ``v1-10-test`` labels are built from branches so they 
change over time. the 1.10.* and in
+  the future ``2.*`` labels are build from git tags and they are "fixed" once 
built.
+* PYTHON_MAJOR_MINOR_VERSION - version of python used to build the image. 
Examples: 3.5, 3.7
+* The ``-ci`` suffix is added for CI images
+* The ``-manifest`` is added for manifest images (see below for explanation of 
manifest images)
+
+Building docker images
+==
+
+The easiest way to build those images is to use ``_.
+
+You can build the CI image using this command:
+
+.. code-block::
+
+  ./breeze build-image
+
+You can build production image using this command:
+
+.. code-block::
+
+  ./breeze build-image --production-image
+
+By adding ``--python `` parameter you can build the
+image version for the chosen python version.
+
+The images are build with default extras - different extras for CI and 
production image and you
+can change the extras via the ``--extras`` parameters. You can see default 
extras used via
+``./breeze flags``.
+
+For example if you want to build python 3.7 version of production image with
+"all" extras installed you should run this command:
+
+.. code-block::
+
+  ./breeze build-image --python 3.7 --extras "all" --production-image
+
+The command that builds the CI image is optimized to minimize the time needed 
to rebuild the image when
+the source code of Airflow evolves. This means that if you already have the 
image locally downloaded and
+built, the scripts will determine whether the rebuild is needed in the first 
place. Then the scripts will
+make sure that minimal number of steps are executed to rebuild parts of the 
image (for example,
+PIP dependencies) and will give you an image consistent with the one used 
during Continuous Integration.
+
+The command that build the production image are optimised for size of the 
image.
+
+In Breeze by default, the images are built using local sources of Apache 
Airflow. However
+you can also build production images from github sources - providing 
``--install-airflow-version``
+parameter to Breeze. This will install airflow inside the production image 
using sources downloaded from
+specified tag or branch. Internally airflow will be installed using the 
command:
+
+.. code-block::
+
+pip install 
https://github.com/apache/airflow/archive/>.tar.gz#egg=apache-airflow \
+   --constraint 
https://raw.githubusercontent.com/apache/airflow//requirements/requirements-python3.7.txt
+
+
+Technical details of Airflow images
+===
+
+The CI image is used by Breeze as shell image but it is also used during CI 
builds on Travis.
+The image is single segment image that contains Airflow installation with 
"all" dependencies installed.
+It is optimised for rebuild speed (AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD flag 
set to "true").
+It installs PIP dependencies from the current branch first - so that any 
changes in setup.py do not trigger
+reinstalling of all dependencies. There is a second step of installation that 
re-installs the dependencies
+from the latest sources so that we are sure that latest dependencies are 
installed.
+
+The production image is a multi-segment image. The first segment 
"airflow-build-image" contains all the
+build essentials and related dependencies that allow to install airflow 
locally. By default the image 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401203890
 
 

 ##
 File path: IMAGES.rst
 ##
 @@ -0,0 +1,389 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+.. contents:: :local:
+
+Airflow docker images
+=
+
+Airflow has two images (build from Dockerfiles):
+
+* CI image (Dockerfile.ci) - used for running tests and local development
+* Production image (Dockerfile) - used to run production-ready Airflow 
installations
+
+Image naming conventions
+
+
+The images are named as follows:
+
+``apache/airflow:-python[-ci][-manifest]``
+
+where:
+
+* BRANCH_OR_TAG - branch or tag used when creating the image. Examples: 
master, v1-10-test, 1.10.10
+  The ``master`` and ``v1-10-test`` labels are built from branches so they 
change over time. the 1.10.* and in
+  the future ``2.*`` labels are build from git tags and they are "fixed" once 
built.
+* PYTHON_MAJOR_MINOR_VERSION - version of python used to build the image. 
Examples: 3.5, 3.7
+* The ``-ci`` suffix is added for CI images
+* The ``-manifest`` is added for manifest images (see below for explanation of 
manifest images)
+
+Building docker images
+==
+
+The easiest way to build those images is to use ``_.
+
+You can build the CI image using this command:
+
+.. code-block::
+
+  ./breeze build-image
+
+You can build production image using this command:
+
+.. code-block::
+
+  ./breeze build-image --production-image
+
+By adding ``--python `` parameter you can build the
+image version for the chosen python version.
+
+The images are build with default extras - different extras for CI and 
production image and you
+can change the extras via the ``--extras`` parameters. You can see default 
extras used via
+``./breeze flags``.
+
+For example if you want to build python 3.7 version of production image with
+"all" extras installed you should run this command:
+
+.. code-block::
+
+  ./breeze build-image --python 3.7 --extras "all" --production-image
+
+The command that builds the CI image is optimized to minimize the time needed 
to rebuild the image when
+the source code of Airflow evolves. This means that if you already have the 
image locally downloaded and
+built, the scripts will determine whether the rebuild is needed in the first 
place. Then the scripts will
+make sure that minimal number of steps are executed to rebuild parts of the 
image (for example,
+PIP dependencies) and will give you an image consistent with the one used 
during Continuous Integration.
+
+The command that build the production image are optimised for size of the 
image.
+
+In Breeze by default, the images are built using local sources of Apache 
Airflow. However
+you can also build production images from github sources - providing 
``--install-airflow-version``
+parameter to Breeze. This will install airflow inside the production image 
using sources downloaded from
+specified tag or branch. Internally airflow will be installed using the 
command:
+
+.. code-block::
+
+pip install 
https://github.com/apache/airflow/archive/>.tar.gz#egg=apache-airflow \
+   --constraint 
https://raw.githubusercontent.com/apache/airflow//requirements/requirements-python3.7.txt
+
+
+Technical details of Airflow images
+===
+
+The CI image is used by Breeze as shell image but it is also used during CI 
builds on Travis.
+The image is single segment image that contains Airflow installation with 
"all" dependencies installed.
+It is optimised for rebuild speed (AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD flag 
set to "true").
+It installs PIP dependencies from the current branch first - so that any 
changes in setup.py do not trigger
+reinstalling of all dependencies. There is a second step of installation that 
re-installs the dependencies
+from the latest sources so that we are sure that latest dependencies are 
installed.
+
+The production image is a multi-segment image. The first segment 
"airflow-build-image" contains all the
+build essentials and related dependencies that allow to install airflow 
locally. By default the image 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401201166
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,373 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
 
 Review comment:
   This is quite an important point, and should be discussed on the mailing 
list.


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401200600
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401198373
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,373 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALPHA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#much smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
 
 Review comment:
   Just noticed this, should have twigged earlier.
   
   I'm not sure how I feel about installing from git, rather than from the 
release artifacts we vote upon.


This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401195975
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
 
 Review comment:
   Official docker images are so you can do `docker pull airflow`? I'm not sure 
AFS would like us to drop the Apache name there.


This is an automated message from the Apache Git Service.
To respond 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r401195449
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
 
 Review comment:
   > This way the Dockerfile becomes a "standalone" Dockerfile.
   >
   > The idea is that once we release Airflow 1.10.10 you should be able to 
just take the Dockerfile (and nothing else) put it in an empty folder and run:
   
   This is different to every other 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400506104
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
 
 Review comment:
   Why is this needed? Isn't `COPY entrypoint.sh /entrypoint` the same and 
easier to reason about?


This is an automated message from the Apache Git Service.
To respond to the message, please log on 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400504559
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400503809
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400503159
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400502678
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo 

[GitHub] [airflow] ashb commented on a change in pull request #7832: Add production image support

[GitBox]

ashb commented on a change in pull request #7832: Add production image support
URL: https://github.com/apache/airflow/pull/7832#discussion_r400502435
 
 

 ##
 File path: Dockerfile
 ##
 @@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# THIS DOCKERFILE IS INTENDED FOR PRODUCTION USE AND DEPLOYMENT.
+# NOTE! IT IS ALFA-QUALITY FOR NOW - WE ARE IN A PROCESS OF TESTING IT
+#
+#
+# This is a multi-segmented image. It actually contains two images:
+#
+# airflow-build-image  - there all airflow dependencies can be installed (and
+#built - for those dependencies that require
+#build essentials). Airflow is installed there with
+#--user switch so that all the dependencies are
+#installed to ${HOME}/.local
+#
+# main - this is the actual production image that is much
+#smaller because it does not contain all the build
+#essentials. Instead the ${HOME}/.local folder
+#is copied from the build-image - this way we have
+#only result of installation and we do not need
+#all the build essentials. This makes the image
+#nuch smaller.
+#
+ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_ORG="apache"
+ARG AIRFLOW_REPO="airflow"
+ARG AIRFLOW_GIT_REFERENCE="master"
+ARG REQUIREMENTS_GIT_REFERENCE="master"
+ARG WWW_FOLDER="www"
+ARG 
AIRFLOW_EXTRAS="async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,ssh,statsd,virtualenv"
+
+ARG AIRFLOW_HOME=/opt/airflow
+ARG AIRFLOW_UID="5"
+ARG AIRFLOW_GID="5"
+
+ARG PIP_VERSION="19.0.2"
+ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
+
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
+ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
+
+##
+# This is the build image where we build all dependencies
+##
+FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+LABEL org.apache.airflow.docker=true
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+
+ARG AIRFLOW_VERSION
+ARG AIRFLOW_ORG
+ARG AIRFLOW_REPO
+ARG AIRFLOW_GIT_REFERENCE
+ARG REQUIREMENTS_GIT_REFERENCE
+ARG WWW_FOLDER
+ARG AIRFLOW_EXTRAS
+
+ARG AIRFLOW_HOME
+ARG AIRFLOW_UID
+ARG AIRFLOW_GID
+
+ARG PIP_VERSION
+ARG CASS_DRIVER_BUILD_CONCURRENCY
+
+ARG PYTHON_MAJOR_MINOR_VERSION
+
+ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
+ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+ENV AIRFLOW_ORG=${AIRFLOW_ORG}
+ENV AIRFLOW_REPO=${AIRFLOW_REPO}
+ENV AIRFLOW_GIT_REFERENCE=${AIRFLOW_GIT_REFERENCE}
+ENV REQUIREMENTS_GIT_REFERENCE=${REQUIREMENTS_GIT_REFERENCE}
+
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+
+ENV AIRFLOW_REPO_URL="https://github.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+ENV 
AIRFLOW_RAW_CONTENT_URL="https://raw.githubusercontent.com/${AIRFLOW_ORG}/${AIRFLOW_REPO};
+
+ENV PIP_VERSION=${PIP_VERSION}
+ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+
+ARG 
AIRFLOW_SOURCES="${AIRFLOW_REPO_URL}/archive/${AIRFLOW_GIT_REFERENCE}.tar.gz#egg=apache-airflow"
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+ARG 
CONSTRAINT_REQUIREMENTS="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ENV CONSTRAINT_REQUIREMENTS=${CONSTRAINT_REQUIREMENTS}
+
+ARG 
ENTRYPOINT_FILE="${AIRFLOW_RAW_CONTENT_URL}/${REQUIREMENTS_GIT_REFERENCE}/entrypoint.sh"
+ENV ENTRYPOINT_FILE="${ENTRYPOINT_FILE}"
+
+# Print versions
+RUN echo "Building airflow-build-image stage" \
+echo "Base image: ${PYTHON_BASE_IMAGE}"; \
+echo "Airflow version: ${AIRFLOW_VERSION}"; \
+echo "Airflow git reference: ${AIRFLOW_GIT_REFERENCE}"; \
+echo