potiuk commented on a change in pull request #4937: [AIRFLOW-4116] Multi-staging includes CI image [Step 2/3] URL: https://github.com/apache/airflow/pull/4937#discussion_r289834068
########## File path: Dockerfile ########## @@ -65,17 +65,126 @@ RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \ # Packages to install \ libsasl2-dev freetds-bin build-essential sasl2-bin \ libsasl2-2 libsasl2-dev libsasl2-modules \ - default-libmysqlclient-dev apt-utils curl rsync netcat locales \ + dirmngr apt-utils curl rsync netcat locales \ freetds-dev libkrb5-dev libssl-dev libffi-dev libpq-dev git \ nodejs gosu sudo \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Install MySQL client from Oracle repositories (Debian installs mariadb) +RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \ + && GNUPGHOME="$(mktemp -d)" \ + && export GNUPGHOME \ + && for KEYSERVER in $(shuf -e \ + ha.pool.sks-keyservers.net \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --keyserver "${KEYSERVER}" --recv-keys "${KEY}" && break || true ; \ + done \ + && gpg --export "${KEY}" > /etc/apt/trusted.gpg.d/mysql.gpg \ + && gpgconf --kill all \ + rm -rf "${GNUPGHOME}"; \ + apt-key list > /dev/null \ + && echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.6" | tee -a /etc/apt/sources.list.d/mysql.list \ + && apt-get update \ + && apt-get install --no-install-recommends -y mysql-client libmysqlclient-dev \ + && apt-get autoremove -yqq --purge \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow +############################################################################################################ +# This is an image with all APT dependencies needed by CI. It is built on top of the airlfow APT image +# Parameters: +# airflow-apt-deps - this is the base image for CI deps image. +############################################################################################################ +FROM airflow-apt-deps as airflow-ci-apt-deps + +SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + +ARG APT_DEPS_IMAGE +ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE} + +RUN echo "${APT_DEPS_IMAGE}" + +# Note the ifs below might be removed if Buildkit will become usable. It should skip building this +# image automatically if it is not used. For now we still go through all layers below but they are empty +# Note missing directories on debian-stretch https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-ci-apt-deps" ]]; then \ + mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + lsb-release gnupg dirmngr openjdk-8-jdk \ + vim tmux less unzip net-tools netcat \ + ldap-utils postgresql-client sqlite3 \ + krb5-user openssh-client openssh-server \ + python-selinux \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + ;\ + fi + +ENV HADOOP_DISTRO=cdh HADOOP_MAJOR=5 HADOOP_DISTRO_VERSION=5.11.0 HADOOP_VERSION=2.6.0 HIVE_VERSION=1.1.0 Review comment: Yeah :). ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
