This is an automated email from the ASF dual-hosted git repository. marcoabreu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new 47e2b89 [MXNET-553] Restructure dockcross dockerfiles to fix caching (#11302) 47e2b89 is described below commit 47e2b89c87bd6332eea6a6f7ea0a9b4550165625 Author: Kellen Sunderland <kellen.sunderl...@gmail.com> AuthorDate: Tue Jun 19 09:29:46 2018 +0200 [MXNET-553] Restructure dockcross dockerfiles to fix caching (#11302) * Add ccache reporting to CI * Restructure dockcross dockerfiles to fix caching --- ci/docker/Dockerfile.build.android_arm64 | 19 ++++++++----------- ci/docker/Dockerfile.build.android_armv7 | 16 ++++++---------- ci/docker/Dockerfile.build.arm64 | 13 +++---------- ci/docker/Dockerfile.build.armv6 | 13 +++---------- ci/docker/Dockerfile.build.armv7 | 13 +++---------- ci/docker/Dockerfile.build.jetson | 12 +++--------- ci/docker/Dockerfile.build.ubuntu_build_cuda | 4 ++-- ci/docker/Dockerfile.build.ubuntu_cpu | 4 ++-- ci/docker/Dockerfile.build.ubuntu_gpu | 4 ++-- .../{ubuntu_ccache.sh => deb_ubuntu_ccache.sh} | 6 ++++-- 10 files changed, 36 insertions(+), 68 deletions(-) diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64 index 4dbc49d..427cc03 100755 --- a/ci/docker/Dockerfile.build.android_arm64 +++ b/ci/docker/Dockerfile.build.android_arm64 @@ -18,22 +18,19 @@ # # Dockerfile to build MXNet for Android ARM64/ARMv8 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/base:latest MAINTAINER Pedro Larroy "pllar...@amazon.com" -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - RUN apt-get update && apt-get install -y \ unzip +WORKDIR /work/deps + +# Build x86 dependencies. +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + +# Setup Android cross-compilation environment. ENV CROSS_TRIPLE=aarch64-linux-android ENV CROSS_ROOT=/usr/${CROSS_TRIPLE} ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \ @@ -63,7 +60,7 @@ ENV ANDROID_NDK_REVISION 15c ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++ -WORKDIR /work/deps +# Build ARM dependencies. COPY install/android_arm64_ndk.sh /work/ RUN /work/android_arm64_ndk.sh COPY install/android_arm64_openblas.sh /work/ diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 index 70bb9fb..859ea02 100755 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android_armv7 @@ -18,19 +18,9 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/base:latest MAINTAINER Pedro Larroy "pllar...@amazon.com" -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - # The cross-compiling emulator RUN apt-get update && apt-get install -y \ qemu-user \ @@ -88,6 +78,12 @@ RUN git clone https://github.com/xianyi/OpenBLAS.git && \ ENV OPENBLAS_ROOT /work/OpenBLAS ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH + +WORKDIR /work/deps + +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + WORKDIR /work ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64 index 2a95007..fd87bf0 100755 --- a/ci/docker/Dockerfile.build.arm64 +++ b/ci/docker/Dockerfile.build.arm64 @@ -18,20 +18,10 @@ # # Dockerfile to build MXNet for ARM64/ARMv8 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837 #FROM dockcross/linux-arm64 FROM mxnetci/dockcross-linux-arm64:05082018 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH aarch64 ENV HOSTCC gcc ENV TARGET ARMV8 @@ -47,5 +37,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/build diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 index 93be540..156c57d 100755 --- a/ci/docker/Dockerfile.build.armv6 +++ b/ci/docker/Dockerfile.build.armv6 @@ -18,18 +18,8 @@ # # Dockerfile to build MXNet for ARMv6 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/linux-armv6 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH armv6l ENV HOSTCC gcc ENV TARGET ARMV6 @@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 index 95e05e7..6316270 100755 --- a/ci/docker/Dockerfile.build.armv7 +++ b/ci/docker/Dockerfile.build.armv7 @@ -18,18 +18,8 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/linux-armv7 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH armv7l ENV HOSTCC gcc ENV TARGET ARMV7 @@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson index 098225e..9cacdc3 100755 --- a/ci/docker/Dockerfile.build.jetson +++ b/ci/docker/Dockerfile.build.jetson @@ -22,20 +22,11 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837 # FROM dockcross/linux-arm64 FROM mxnetci/dockcross-linux-arm64:05082018 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH aarch64 ENV HOSTCC gcc ENV TARGET ARMV8 @@ -51,6 +42,9 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + # Setup CUDA build env (including configuring and copying nvcc) COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda ENV TARGET_ARCH aarch64 diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda index a87651a..51b398c 100755 --- a/ci/docker/Dockerfile.build.ubuntu_build_cuda +++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda @@ -27,8 +27,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index f82ee75..57cf1e9 100755 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -24,8 +24,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu index 5f4bfc5..1696c18 100755 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -24,8 +24,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/install/ubuntu_ccache.sh b/ci/docker/install/deb_ubuntu_ccache.sh similarity index 85% rename from ci/docker/install/ubuntu_ccache.sh rename to ci/docker/install/deb_ubuntu_ccache.sh index 4a6b63f..31cd5c8 100755 --- a/ci/docker/install/ubuntu_ccache.sh +++ b/ci/docker/install/deb_ubuntu_ccache.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -# Script to build ccache for ubuntu based images +# Script to build ccache for debian and ubuntu based images. set -ex @@ -43,7 +43,9 @@ git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git cd ccache ./autogen.sh -./configure +# Manually specify x86 gcc versions so that this script remains compatible with dockcross (which uses an ARM based gcc +# by default). +CC=/usr/bin/gcc CXX=/usr/bin/g++ ./configure # Don't build documentation #11214 #perl -pi -e 's!\s+\Q$(installcmd) -d $(DESTDIR)$(mandir)/man1\E!!g' Makefile