This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 1650e2f39449a1b89d4c3abba1106b5ad6bf824a Author: Grant Henke <[email protected]> AuthorDate: Wed Dec 18 08:25:11 2019 -0600 KUDU-3005: Reduce size of kudu-python docker image This patch breaks out the python environment bootstrapping from the dev environment bootstrapping. This allows us to use the runtime image for the kudu-python docker image instead of the giant dev image. The result of this change is a 440MB uncompressed image instead of a 1.92GB uncompressed image. I manually tested the image by running the python example against a quickstart cluster: https://github.com/apache/kudu/blob/master/examples/python/basic-python-example/basic_example.py Change-Id: I159907c4a85e919d98c92df97cd44d0300d1cd29 Reviewed-on: http://gerrit.cloudera.org:8080/14984 Reviewed-by: Adar Dembo <[email protected]> Reviewed-by: Andrew Wong <[email protected]> Tested-by: Kudu Jenkins --- .dockerignore | 1 + docker/Dockerfile | 21 +++++++--- docker/bootstrap-dev-env.sh | 32 --------------- docker/bootstrap-python-env.sh | 93 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 37 deletions(-) diff --git a/.dockerignore b/.dockerignore index 007dd33..1379088 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,6 +30,7 @@ # Docker files. !docker/bootstrap-dev-env.sh +!docker/bootstrap-python-env.sh !docker/bootstrap-runtime-env.sh !docker/kudu-entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 266c372..444dd32 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -62,7 +62,11 @@ ARG BASE_OS FROM $BASE_OS as dev COPY ./docker/bootstrap-dev-env.sh / -RUN ./bootstrap-dev-env.sh && rm bootstrap-dev-env.sh +COPY ./docker/bootstrap-python-env.sh / +RUN ./bootstrap-dev-env.sh \ + && ./bootstrap-python-env.sh \ + && rm bootstrap-dev-env.sh \ + && rm bootstrap-python-env.sh # Common label arguments. # VCS_REF is not specified to improve docker caching. @@ -221,13 +225,20 @@ LABEL name="Apache Kudu Build" \ # ---- Kudu Python ---- # Builds a runtime image with the Kudu python client pre-installed. # -FROM dev AS kudu-python +FROM runtime AS kudu-python + +COPY ./docker/bootstrap-python-env.sh / +RUN ./bootstrap-python-env.sh \ + && rm bootstrap-python-env.sh + +ARG INSTALL_DIR="/opt/kudu" ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64 WORKDIR $INSTALL_DIR/python # Copy the requirements file. COPY ./python/requirements.txt requirements.txt -COPY --from=build /usr/local /usr/local/ +COPY --from=build /usr/local/lib/libkudu_client* /usr/local/lib/ +COPY --from=build /usr/local/include/kudu /usr/local/include/kudu COPY --from=build /kudu/python/dist/kudu-python-*.tar.gz . RUN pip install -r requirements.txt \ && rm -rf requirements.txt \ @@ -241,8 +252,8 @@ ARG VCS_TYPE ARG VCS_URL ARG VERSION -LABEL org.label-schema.name="Apache Kudu Python Client Base" \ - org.label-schema.description="Builds a runtime image with the Kudu python client pre-installed." \ +LABEL org.label-schema.name="Apache Kudu Python Client" \ + org.label-schema.description="An image with the Kudu Python client pre-installed." \ # Common labels. org.label-schema.dockerfile=$DOCKERFILE \ org.label-schema.maintainer=$MAINTAINER \ diff --git a/docker/bootstrap-dev-env.sh b/docker/bootstrap-dev-env.sh index 4cbddcf..62754ba 100755 --- a/docker/bootstrap-dev-env.sh +++ b/docker/bootstrap-dev-env.sh @@ -25,29 +25,6 @@ set -xe -function install_python_packages() { - PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2) - PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) - PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) - - # We use get-pip.py to bootstrap pip outside of system packages. - # This prevents issues with the platform package manager knowing - # about only some of the python packages. - if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then - # Beginning with pip 10, Python 2.6 is no longer supported. - curl https://bootstrap.pypa.io/2.6/get-pip.py | python - else - # Use a stable version of pip that works with the remaining - # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4, - # which is the version of Python 3 shipped with Ubuntu 14.04. - curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0" - fi - pip install --upgrade \ - cython \ - setuptools \ - setuptools_scm -} - # Install the prerequisite libraries, if they are not installed. # CentOS/RHEL if [[ -f "/usr/bin/yum" ]]; then @@ -92,11 +69,6 @@ if [[ -f "/usr/bin/yum" ]]; then ruby-devel \ zlib-devel - # Install python development packages. - yum install -y epel-release - yum install -y python-devel - install_python_packages - # To build on a version older than 7.0, the Red Hat Developer Toolset # must be installed (in order to have access to a C++11 capable compiler). OS_MAJOR_VERSION=$(lsb_release -rs | cut -f1 -d.) @@ -180,10 +152,6 @@ elif [[ -f "/usr/bin/apt-get" ]]; then xsltproc \ zlib1g-dev - # Install python development packages. - apt-get install -y --no-install-recommends python-dev - install_python_packages - # Reduce the image size by cleaning up after the install. apt-get clean rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docker/bootstrap-python-env.sh b/docker/bootstrap-python-env.sh new file mode 100755 index 0000000..ac7f14b --- /dev/null +++ b/docker/bootstrap-python-env.sh @@ -0,0 +1,93 @@ +#!/bin/bash +########################################################## +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This script handles bootstrapping a base OS for +# the Apache Kudu base docker images. +# +########################################################## + +set -xe + +function install_python_packages() { + PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2) + PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) + PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) + + # We use get-pip.py to bootstrap pip outside of system packages. + # This prevents issues with the platform package manager knowing + # about only some of the python packages. + if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then + # Beginning with pip 10, Python 2.6 is no longer supported. + curl https://bootstrap.pypa.io/2.6/get-pip.py | python + else + # Use a stable version of pip that works with the remaining + # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4, + # which is the version of Python 3 shipped with Ubuntu 14.04. + curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0" + fi + pip install --upgrade \ + cython \ + setuptools \ + setuptools_scm +} + +# Install the prerequisite libraries, if they are not installed. +# CentOS/RHEL +if [[ -f "/usr/bin/yum" ]]; then + # Update the repo. + yum update -y + + # Install curl, used when installing pip. + yum install -y ca-certificates curl + + # Install python development packages. + yum install -y epel-release + # g++ is required to check for int128 support in setup.py. + yum install -y gcc gcc-c++ python-devel + install_python_packages + + # Reduce the image size by cleaning up after the install. + yum clean all + rm -rf /var/cache/yum /tmp/* /var/tmp/* +# Ubuntu/Debian +elif [[ -f "/usr/bin/apt-get" ]]; then + # Ensure the Debian frontend is noninteractive. + export DEBIAN_FRONTEND=noninteractive + + # Update the repo. + apt-get update -y + + # Install curl, used when installing pip. + apt-get install -y --no-install-recommends ca-certificates curl + + # Install python development packages. + # g++ is required to check for int128 support in setup.py. + apt-get install -y --no-install-recommends g++ python-dev + install_python_packages + + # Reduce the image size by cleaning up after the install. + apt-get clean + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + + unset DEBIAN_FRONTEND +else + echo "Unsupported OS" + exit 1 +fi
