mbaijal closed pull request #12791: [MXNET-1093] Add python3 Docker images for each MXNet release URL: https://github.com/apache/incubator-mxnet/pull/12791
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index eb437f3c233..0858e99e2d7 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for CPU FROM ubuntu:16.04 +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet +RUN pip install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index 043932ff7c8..dbb7d29f503 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet CPU with MKL FROM ubuntu:16.04 +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-mkl +RUN pip install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 index 8c83ece434a..cb22721f48b 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:8.0-cudnn5-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu80 +RUN pip install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl index a057c1d20cb..eda96c90d18 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:8.0-cudnn5-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu80mkl +RUN pip install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 index 1e3d9869ac6..cd36b8afdbd 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.0-cudnn7-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu90 +RUN pip install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl index d82abd7cf52..0b274c4e0d3 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.0-cudnn7-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu90mkl +RUN pip install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 index ba5c54a2a2a..2c43187faf7 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.2-cudnn7-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu92 +RUN pip install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl index 96a943980b5..db204897ef6 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.2-cudnn7-devel +ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py -RUN pip install mxnet-cu92mkl +RUN pip install mxnet-cu92mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu new file mode 100644 index 00000000000..8ad9950a2f2 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for CPU + +FROM ubuntu:16.04 +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl new file mode 100644 index 00000000000..c6312891c6e --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet CPU with MKL + +FROM ubuntu:16.04 +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 new file mode 100644 index 00000000000..58af6bfeb27 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:8.0-cudnn5-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl new file mode 100644 index 00000000000..059f002c856 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:8.0-cudnn5-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 new file mode 100644 index 00000000000..a860de91805 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.0-cudnn7-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl new file mode 100644 index 00000000000..c0b6145b28e --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.0-cudnn7-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 new file mode 100644 index 00000000000..6d877961db9 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl new file mode 100644 index 00000000000..b73df97e595 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -0,0 +1,29 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.2-cudnn7-devel +ARG version + +RUN apt-get update && \ + apt-get install -y wget python3-dev gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install mxnet-cu92mkl==$version diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index f806d5d6f45..d84d23955b1 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,30 +1,40 @@ # Release Python Docker Images for MXNet The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. -It uses the appropriate pip binaries to build different docker images as - -* cpu -* cpu_mkl -* latest (same as cpu) -* gpu_cu90 -* gpu_cu90_mkl -* gpu (same as gpu_cu90) -* gpu_cu80 -* gpu_cu80_mkl -* gpu_cu92 -* gpu_cu92_mkl - - -** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. +It uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as - +* {version}_cpu +* {version}_cpu_mkl +* {version}_gpu_cu90 +* {version}_gpu_cu90_mkl +* {version}_gpu_cu80 +* {version}_gpu_cu80_mkl +* {version}_gpu_cu92 +* {version}_gpu_cu92_mkl +* {version}_cpu_py3 +* {version}_cpu_mkl_py3 +* {version}_gpu_cu90_py3 +* {version}_gpu_cu90_mkl_py3 +* {version}_gpu_cu80_py3 +* {version}_gpu_cu80_mkl_py3 +* {version}_gpu_cu92_py3 +* {version}_gpu_cu92_mkl_py3 + +And the following tags will be available without the version string in the image name (for Benchmarking and other use cases): +* latest (same as {version}_cpu) +* gpu (same as {version}_gpu_cu90) +* latest_cpu_mkl_py2 (same as {version}_cpu_mkl) +* latest_cpu_mkl_py3 (same as {version}_cpu_mkl_py3) +* latest_gpu_mkl_py2 (same as {version}_gpu_cu90_mkl) +* latest_gpu_mkl_py3 (same as {version}_gpu_cu90_mkl_py3) Refer: https://pypi.org/project/mxnet/ -### Usage -`./build_python_dockerfile.sh <mxnet_version> <path_to_cloned_mxnet_repo>` +### Using the Build Script +`./build_python_dockerfile.sh <mxnet_version> <pip_tag> <path_to_cloned_mxnet_repo>` For example: -`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` +`./build_python_dockerfile.sh 1.3.0 1.3.0.post0 ~/build-docker/incubator-mxnet` -** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run * [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) @@ -39,3 +49,10 @@ Credentials can be provided in the following ways: * **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime - * $MXNET_DOCKERHUB_PASSWORD * $MXNET_DOCKERHUB_USERNAME + + +### Using the Docker Images +* The MXNet Python Docker images can be found here: https://hub.docker.com/r/mxnet/python/ + +* Docker Pull Command: `docker pull mxnet/python:<image_tag>` +* Get started: `docker run -it mxnet/python:<image_tag> bash` diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 24a44c28970..d0f24c8d6a5 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -17,20 +17,19 @@ # specific language governing permissions and limitations # under the License. -set -e - # Check Params programname=$0 function usage { - echo "usage: $programname [version] [path]" + echo "usage: $programname [version] [pip_tag] [path]" echo " [version] Mxnet Version to build" + echo " [pip_tag] Pip Tag to use" echo " [path] Path to MXNet repository (to run tests)" echo " " exit 1 } -if [ $# -le 1 ] || [ $# -ge 3 ] +if [ $# -le 2 ] || [ $# -ge 4 ] then usage exit 1 @@ -39,29 +38,37 @@ fi # Two params provided echo "Building Docker Images for Apache MXNet (Incubating) v$1" mxnet_version="${1}" -test_dir="${2}" +pip_tag="${2}" +test_dir="${3}" -docker_build_image(){ - echo "Building docker image mxnet/python:${1}" - docker build -t mxnet/python:${1} -f ${2} . -} +# Remove the logs directory if it already exists else it may fail due to old logs. +LOGDIR=~/temp/docker_logs +if [ -d "${LOGDIR}" ]; then + rm -rf ${LOGDIR} +fi + +# Create ~/temp if it does not exist +mkdir -p ~/temp +mkdir ${LOGDIR} -docker_tag_image(){ - docker tag mxnet/python:${1} mxnet/python:${2} -} +# Functions docker_test_image_cpu(){ - echo "Running tests on mxnet/python:${1}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" + image_tag="${1}" + python_version="${2}" + echo "Running tests on mxnet/python:${image_tag}" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py" } docker_test_image_gpu(){ + image_tag="${1}" + python_version="${2}" echo "Running tests on mxnet/python:${1}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py --gpu" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py --gpus 0,1,2,3" } # if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login @@ -79,59 +86,132 @@ docker_account_logout(){ } docker_push_image(){ - docker push mxnet/python:${1} + image_tag="${1}" + docker push mxnet/python:${image_tag} } +docker_generate_image_cpu(){ + image_tag="${1}" + dockerfile="${2}" + python_version="${3}" + echo "Building docker image mxnet/python:${image_tag}" + docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . + docker_test_image_cpu ${image_tag} ${python_version} +} -# Build and Test dockerfiles - CPU -docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" -docker_test_image_cpu "${mxnet_version}_cpu" - -docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" -docker_test_image_cpu "${mxnet_version}_cpu_mkl" - -docker_tag_image "${mxnet_version}_cpu" "latest" -docker_test_image_cpu "latest" - - -#Build and Test dockerfiles - GPU -docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" -docker_test_image_gpu "${mxnet_version}_gpu_cu90" - -docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl" +docker_tag_image_cpu(){ + original_tag="${1}" + image_tag="${2}" + python_version="${3}" + docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} + docker_test_image_cpu ${image_tag} ${python_version} +} -docker_tag_image "${mxnet_version}_gpu_cu90" "gpu" -docker_test_image_gpu "gpu" +docker_generate_image_gpu(){ + image_tag="${1}" + dockerfile="${2}" + python_version="${3}" + echo "Building docker image mxnet/python:${1}" + docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . + docker_test_image_gpu ${image_tag} ${python_version} +} -docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" -docker_test_image_gpu "${mxnet_version}_gpu_cu80" +docker_tag_image_gpu(){ + original_tag="${1}" + image_tag="${2}" + python_version="${3}" + docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} + docker_test_image_gpu ${image_tag} ${python_version} +} -docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl" +check_errors(){ + egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* + if [ $? -eq 0 ]; then + echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." + exit 1 + else + echo "Success: No errors found" + fi +} -docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" -docker_test_image_gpu "${mxnet_version}_gpu_cu92" +# Build and Test dockerfiles - CPU +docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 & -docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl" +#Build and Test dockerfiles - GPU +docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" > ${LOGDIR}/docker_gpu_cu92_mkl.out 2>&1 + +echo "Waiting for MXNet Python2 Docker Images to Build" +wait + +# Build and Test Python3 dockerfiles - CPU +docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" > ${LOGDIR}/docker_cpu_py3.out 2>&1 & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" > ${LOGDIR}/docker_cpu_mkl_py3.out 2>&1 & + +#Build and Test Python3 dockerfiles - GPU +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" "python3" > ${LOGDIR}/docker_gpu_cu90_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" "python3" > ${LOGDIR}/docker_gpu_cu90_mkl_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" "python3" > ${LOGDIR}/docker_gpu_cu80_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" "python3" > ${LOGDIR}/docker_gpu_cu80_mkl_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" > ${LOGDIR}/docker_gpu_cu92_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" > ${LOGDIR}/docker_gpu_cu92_mkl_py3.out 2>&1 + +echo "Waiting for MXNet Python3 Docker Images to Build" +wait + +echo "Re-Tag 6 images with version-free names (for Benchmarking) - only after previous builds complete. " +docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu.out 2>&1 & +docker_tag_image_cpu "${mxnet_version}_cpu_mkl" "latest_cpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 & +docker_tag_image_cpu "${mxnet_version}_cpu_mkl_py3" "latest_cpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl" "latest_gpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_gpu_mkl_py2.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "latest_gpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_gpu_mkl_py3.out 2>&1 +wait + +# Parse all the docker logfiles to make sure there is no error. Fail script if error is found. +check_errors # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" docker_account_login +# Python2 docker_push_image "${mxnet_version}_cpu" docker_push_image "${mxnet_version}_cpu_mkl" -docker_push_image "latest" docker_push_image "${mxnet_version}_gpu_cu90" docker_push_image "${mxnet_version}_gpu_cu90_mkl" -docker_push_image "gpu" docker_push_image "${mxnet_version}_gpu_cu80" docker_push_image "${mxnet_version}_gpu_cu80_mkl" docker_push_image "${mxnet_version}_gpu_cu92" docker_push_image "${mxnet_version}_gpu_cu92_mkl" +# Python3 +docker_push_image "${mxnet_version}_cpu_py3" +docker_push_image "${mxnet_version}_cpu_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu90_py3" +docker_push_image "${mxnet_version}_gpu_cu90_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu80_py3" +docker_push_image "${mxnet_version}_gpu_cu80_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu92_py3" +docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3" + +docker_push_image "latest" +docker_push_image "gpu" +docker_push_image "latest_cpu_mkl_py2" +docker_push_image "latest_cpu_mkl_py3" +docker_push_image "latest_gpu_mkl_py2" +docker_push_image "latest_gpu_mkl_py3" + + docker_account_logout echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/"; + +#Delete the log directory since everything succeeded: +rm -rf ${LOGDIR} \ No newline at end of file ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
