This is an automated email from the ASF dual-hosted git repository. marcoabreu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new 8ebc050 [MXNET-416] Add docker cache (#10917) 8ebc050 is described below commit 8ebc050e270498844247af8196489d0d3cdfc300 Author: Marco de Abreu <marcoab...@users.noreply.github.com> AuthorDate: Wed May 16 14:31:14 2018 +0200 [MXNET-416] Add docker cache (#10917) * Initial work for distributed docker cache * Fix Android ARMv7 dockerfile and improve docker cache script * add docker image id * Add s3 upload * Add s3 upload * Add cache download * Extend script to build all dockerfiles * Add loading of docker cache before build * Switch to multiprocessing * Add jenkins job * Improve Jenkinsfile * Reorganize Jenkinsfile * Preserve entire image chain * Fix bugs in cache generation * Address review comments * Address review comments * Remove unused Dockerfiles (Amazon Linux) * Address review comments --- Jenkinsfile | 98 +++++---- ci/Jenkinsfile_docker_cache | 81 ++++++++ ci/__init__.py | 0 ci/build.py | 51 ++++- ci/docker/Dockerfile.build.amzn_linux_cpu | 44 ---- ci/docker/Dockerfile.build.android_armv7 | 11 +- ci/docker/Dockerfile.build.centos7_cpu | 4 +- ci/docker/Dockerfile.build.centos7_gpu | 4 +- ci/docker/Dockerfile.build.ubuntu_build_cuda | 3 +- ci/docker/Dockerfile.build.ubuntu_cpu | 4 +- ci/docker/Dockerfile.build.ubuntu_gpu | 4 +- ci/docker/install/amzn_linux_core.sh | 45 ----- ci/docker/install/amzn_linux_julia.sh | 29 --- ci/docker/install/amzn_linux_library.sh | 26 --- ci/docker/install/amzn_linux_maven.sh | 27 --- ci/docker/install/amzn_linux_openblas.sh | 29 --- ci/docker/install/amzn_linux_opencv.sh | 33 --- ci/docker/install/amzn_linux_python2.sh | 36 ---- ci/docker/install/amzn_linux_python3.sh | 44 ---- ci/docker/install/amzn_linux_testdeps.sh | 27 --- ci/docker_cache.py | 292 +++++++++++++++++++++++++++ ci/docker_cache_requirements | 8 + 22 files changed, 493 insertions(+), 407 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f18d7a9..4d8dfb7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,24 +92,34 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum """ } +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { + def command = "ci/build.py --download-docker-cache --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') + command = command.replaceAll('%PLATFORM%', platform) + command = command.replaceAll('%FUNCTION_NAME%', function_name) + command = command.replaceAll('%SHARED_MEM%', shared_mem) + + sh command +} + // Python unittest for CPU // Python 2 def python2_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_cpu" + docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false) } } // Python 3 def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu" + docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false) } } def python3_ut_mkldnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu_mkldnn" + docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false) } } @@ -118,14 +128,14 @@ def python3_ut_mkldnn(docker_container_name) { // Python 2 def python2_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py --nvidiadocker --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_gpu" + docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true) } } // Python 3 def python3_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py --nvidiadocker --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_gpu" + docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true) } } @@ -134,7 +144,7 @@ try { node('mxnetlinux-cpu') { ws('workspace/sanity') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check" + docker_run('ubuntu_cpu', 'sanity_check', false) } } } @@ -145,7 +155,7 @@ try { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh build_centos7_cpu" + docker_run('centos7_cpu', 'build_centos7_cpu', false) pack_lib('centos7_cpu') } } @@ -156,7 +166,7 @@ try { ws('workspace/build-centos7-mkldnn') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh build_centos7_mkldnn" + docker_run('centos7_cpu', 'build_centos7_mkldnn', false) pack_lib('centos7_mkldnn') } } @@ -167,7 +177,7 @@ try { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform centos7_gpu /work/runtime_functions.sh build_centos7_gpu" + docker_run('centos7_gpu', 'build_centos7_gpu', false) pack_lib('centos7_gpu') } } @@ -178,7 +188,7 @@ try { ws('workspace/build-cpu-openblas') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) pack_lib('cpu', mx_dist_lib) } } @@ -189,7 +199,7 @@ try { ws('workspace/build-cpu-clang39') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false) } } } @@ -199,7 +209,7 @@ try { ws('workspace/build-cpu-clang50') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false) } } } @@ -209,7 +219,7 @@ try { ws('workspace/build-cpu-mkldnn-clang39') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39_mkldnn" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false) pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib) } } @@ -220,7 +230,7 @@ try { ws('workspace/build-cpu-mkldnn-clang50') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50_mkldnn" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false) pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib) } } @@ -231,7 +241,7 @@ try { ws('workspace/build-mkldnn-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn" + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false) pack_lib('mkldnn_cpu', mx_mkldnn_lib) } } @@ -242,7 +252,7 @@ try { ws('workspace/build-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_mkldnn" + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) pack_lib('mkldnn_gpu', mx_mkldnn_lib) } } @@ -253,7 +263,7 @@ try { ws('workspace/build-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7" + docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) pack_lib('gpu', mx_dist_lib) stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' @@ -274,7 +284,7 @@ try { ws('workspace/amalgamationmin') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation_min" + docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false) } } } @@ -284,7 +294,7 @@ try { ws('workspace/amalgamation') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation" + docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false) } } } @@ -295,7 +305,7 @@ try { ws('workspace/build-cmake-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn" //build_cuda + docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false) pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) } } @@ -306,7 +316,7 @@ try { ws('workspace/build-cmake-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake" //build_cuda + docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) pack_lib('cmake_gpu', mx_cmake_lib) } } @@ -424,7 +434,7 @@ try { ws('workspace/build-jetson-armv8') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform jetson /work/runtime_functions.sh build_jetson" + docker_run('jetson', 'build_jetson', false) } } } @@ -434,7 +444,7 @@ try { ws('workspace/build-raspberry-armv7') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform armv7 /work/runtime_functions.sh build_armv7" + docker_run('armv7', 'build_armv7', false) } } } @@ -444,7 +454,7 @@ try { ws('workspace/build-raspberry-armv6') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform armv6 /work/runtime_functions.sh build_armv6" + docker_run('armv6', 'build_armv6', false) } } } @@ -494,7 +504,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu', mx_lib) - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python2_quantization_gpu" + docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true) } } } @@ -505,7 +515,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu', mx_lib) - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_quantization_gpu" + docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true) } } } @@ -552,7 +562,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('centos7_cpu') - sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh unittest_centos7_cpu" + docker_run('centos7_cpu', 'unittest_centos7_cpu', false) } } } @@ -563,7 +573,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('centos7_gpu') - sh "ci/build.py --nvidiadocker --platform centos7_gpu /work/runtime_functions.sh unittest_centos7_gpu" + docker_run('centos7_gpu', 'unittest_centos7_gpu', true) } } } @@ -574,7 +584,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cpu', mx_dist_lib) - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala" + docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) } } } @@ -585,7 +595,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu', mx_dist_lib) - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_scala" + docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_scala', true) } } } @@ -596,7 +606,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cpu') - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl" + docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) } } } @@ -607,7 +617,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl" + docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) } } } @@ -618,7 +628,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cmake_gpu', mx_cmake_lib) - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp" + docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) } } } @@ -629,7 +639,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp" + docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) } } } @@ -640,7 +650,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cpu') - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_R" + docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) } } } @@ -651,7 +661,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_R" + docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) } } } @@ -756,7 +766,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('cpu') - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh integrationtest_ubuntu_cpu_onnx" + docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) } } } @@ -767,7 +777,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_python" + docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) } } } @@ -778,7 +788,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_caffe" + docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) } } } @@ -799,7 +809,7 @@ try { unstash 'cpp_mlp_gpu' unstash 'cpp_test_score' unstash 'cpp_test_optimizer' - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_cpp_package" + docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) } } } @@ -810,7 +820,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_dist_kvstore" + docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) } } } @@ -821,7 +831,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python2_gpu" + docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python2_gpu', true, '3g') } } } @@ -832,7 +842,7 @@ try { timeout(time: max_time, unit: 'MINUTES') { init_git() unpack_lib('gpu') - sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python3_gpu" + docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python3_gpu', true, '3g') } } } @@ -844,7 +854,7 @@ try { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { init_git() - sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs" + docker_run('ubuntu_cpu', 'deploy_docs', false) sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" } } diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache new file mode 100644 index 0000000..8a0428b --- /dev/null +++ b/ci/Jenkinsfile_docker_cache @@ -0,0 +1,81 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkins pipeline to generate the centralized docker cache +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +total_timeout = 120 +git_timeout = 15 +// assign any caught errors here +err = null + +// initialize source codes +def init_git() { + deleteDir() + retry(5) { + try { + // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of + // retries as this will increase the amount of requests and worsen the throttling + timeout(time: git_timeout, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init --recursive' + sh 'git clean -x -d -f' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes with ${exc}" + sleep 2 + } + } +} + + +try { + stage("Docker cache build & publish") { + node('mxnetlinux-cpu') { + ws('workspace/docker_cache') { + timeout(time: total_timeout, unit: 'MINUTES') { + init_git() + sh "ci/docker_cache.py --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET}" + } + } + } + } + + // set build status to success at the end + currentBuild.result = "SUCCESS" +} catch (caughtError) { + node("mxnetlinux-cpu") { + sh "echo caught ${caughtError}" + err = caughtError + currentBuild.result = "FAILURE" + } +} finally { + node("mxnetlinux-cpu") { + // Only send email if master failed + if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } +} diff --git a/ci/__init__.py b/ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ci/build.py b/ci/build.py index 6d8d014..6b1d23e 100755 --- a/ci/build.py +++ b/ci/build.py @@ -33,6 +33,7 @@ import re import shutil import subprocess import sys +import docker_cache from copy import deepcopy from itertools import chain from subprocess import call, check_call @@ -61,17 +62,44 @@ def get_docker_binary(use_nvidia_docker: bool) -> str: def build_docker(platform: str, docker_binary: str) -> None: - """Build a container for the given platform""" + """ + Build a container for the given platform + :param platform: Platform + :param docker_binary: docker binary to use (docker/nvidia-docker) + :return: Id of the top level image + """ + tag = get_docker_tag(platform) logging.info("Building container tagged '%s' with %s", tag, docker_binary) cmd = [docker_binary, "build", "-f", get_dockerfile(platform), + "--rm=false", # Keep intermediary layers to prime the build cache "--build-arg", "USER_ID={}".format(os.getuid()), + "--cache-from", tag, "-t", tag, "docker"] logging.info("Running command: '%s'", ' '.join(cmd)) check_call(cmd) + # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the + # check_call would have failed + image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag) + if not image_id: + raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag)) + return image_id + + +def _get_local_image_id(docker_binary, docker_tag): + """ + Get the image id of the local docker layer with the passed tag + :param docker_tag: docker tag + :return: Image id as string or None if tag does not exist + """ + cmd = [docker_binary, "images", "-q", docker_tag] + image_id_b = subprocess.check_output(cmd) + image_id = image_id_b.decode('utf-8').strip() + return image_id + def get_mxnet_root() -> str: curpath = os.path.abspath(os.path.dirname(__file__)) @@ -123,6 +151,7 @@ def container_run(platform: str, if not dry_run and ret != 0: logging.error("Running of command in container failed (%s): %s", ret, cmd) logging.error("You can try to get into the container by using the following command: %s", docker_run_cmd) + raise subprocess.CalledProcessError(ret, cmd) return docker_run_cmd @@ -131,7 +160,6 @@ def container_run(platform: str, def list_platforms() -> str: print("\nSupported platforms:\n{}".format('\n'.join(get_platforms()))) - def main() -> int: # We need to be in the same directory than the script so the commands in the dockerfiles work as # expected. But the script can be invoked from a different path @@ -180,6 +208,14 @@ def main() -> int: help="go in a shell inside the container", action='store_true') + parser.add_argument("--download-docker-cache", + help="Download the docker cache from our central repository instead of rebuilding locally", + action='store_true') + + parser.add_argument("--docker-cache-bucket", + help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", + type=str) + parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) @@ -194,12 +230,15 @@ def main() -> int: list_platforms() elif args.platform: platform = args.platform + tag = get_docker_tag(platform) + if args.download_docker_cache: + logging.info('Docker cache download is enabled') + docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: - logging.warn("Container was just built. Exiting due to build-only.") + logging.warning("Container was just built. Exiting due to build-only.") return 0 - tag = get_docker_tag(platform) if command: container_run(platform, docker_binary, shared_memory_size, command) elif args.print_docker_run: @@ -216,6 +255,10 @@ def main() -> int: logging.info("Building for all architectures: {}".format(platforms)) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: + if args.download_docker_cache: + tag = get_docker_tag(platform) + logging.info('Docker cache download is enabled') + docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: continue diff --git a/ci/docker/Dockerfile.build.amzn_linux_cpu b/ci/docker/Dockerfile.build.amzn_linux_cpu deleted file mode 100755 index 7d6f223..0000000 --- a/ci/docker/Dockerfile.build.amzn_linux_cpu +++ /dev/null @@ -1,44 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build and run MXNet for Amazon Linux on CPU - -FROM amazonlinux - -WORKDIR /work/deps -COPY install/amzn_linux_core.sh /work/ -RUN /work/amzn_linux_core.sh -COPY install/amzn_linux_opencv.sh /work/ -RUN /work/amzn_linux_opencv.sh -COPY install/amzn_linux_openblas.sh /work/ -RUN /work/amzn_linux_openblas.sh -COPY install/amzn_linux_python2.sh /work/ -RUN /work/amzn_linux_python2.sh -COPY install/amzn_linux_python3.sh /work/ -RUN /work/amzn_linux_python3.sh -COPY install/amzn_linux_testdeps.sh /work/ -RUN /work/amzn_linux_testdeps.sh -COPY install/amzn_linux_julia.sh /work/ -RUN /work/amzn_linux_julia.sh -COPY install/amzn_linux_maven.sh /work/ -RUN /work/amzn_linux_maven.sh -COPY install/amzn_linux_library.sh /work/ -RUN /work/amzn_linux_library.sh -WORKDIR /work/mxnet - -COPY runtime_functions.sh /work/ \ No newline at end of file diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 index 0074c1f..c22e000 100755 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android_armv7 @@ -84,13 +84,6 @@ ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang ENV CXX /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang++ ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0" -# Build MXNet -ADD mxnet mxnet -ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk -RUN cd mxnet && \ - make -j$(nproc) $BUILD_OPTS +WORKDIR /work/mxnet -WORKDIR /work/build/ -RUN cp /work/mxnet/lib/* . - -# TODO: Bring this into the new format \ No newline at end of file +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu index a44d646..92314fa 100755 --- a/ci/docker/Dockerfile.build.centos7_cpu +++ b/ci/docker/Dockerfile.build.centos7_cpu @@ -20,8 +20,6 @@ FROM centos:7 -ARG USER_ID=0 - WORKDIR /work/deps COPY install/centos7_core.sh /work/ @@ -30,6 +28,8 @@ COPY install/centos7_python.sh /work/ RUN /work/centos7_python.sh COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh + +ARG USER_ID=0 COPY install/centos7_adduser.sh /work/ RUN /work/centos7_adduser.sh diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu index 4dcf5bf..2d28170 100755 --- a/ci/docker/Dockerfile.build.centos7_gpu +++ b/ci/docker/Dockerfile.build.centos7_gpu @@ -20,14 +20,14 @@ FROM nvidia/cuda:9.1-cudnn7-devel-centos7 -ARG USER_ID=0 - WORKDIR /work/deps COPY install/centos7_core.sh /work/ RUN /work/centos7_core.sh COPY install/centos7_python.sh /work/ RUN /work/centos7_python.sh + +ARG USER_ID=0 COPY install/centos7_adduser.sh /work/ RUN /work/centos7_adduser.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda index 9156d6f..4d3c466 100755 --- a/ci/docker/Dockerfile.build.ubuntu_build_cuda +++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda @@ -23,8 +23,6 @@ FROM nvidia/cuda:9.1-cudnn7-devel -ARG USER_ID=0 - WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ @@ -48,6 +46,7 @@ COPY install/ubuntu_nvidia.sh /work/ RUN /work/ubuntu_nvidia.sh # Keep this at the end since this command is not cachable +ARG USER_ID=0 COPY install/ubuntu_adduser.sh /work/ RUN /work/ubuntu_adduser.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index f706f88..2dc7ef1 100755 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -20,8 +20,6 @@ FROM ubuntu:16.04 -ARG USER_ID=0 - WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ @@ -44,6 +42,8 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ RUN /work/ubuntu_docs.sh + +ARG USER_ID=0 COPY install/ubuntu_adduser.sh /work/ RUN /work/ubuntu_adduser.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu index 547f984..1097172 100755 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -20,8 +20,6 @@ FROM nvidia/cuda:9.1-cudnn7-devel -ARG USER_ID=0 - WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ @@ -50,6 +48,8 @@ COPY install/ubuntu_docs.sh /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ RUN /work/ubuntu_tutorials.sh + +ARG USER_ID=0 COPY install/ubuntu_adduser.sh /work/ RUN /work/ubuntu_adduser.sh diff --git a/ci/docker/install/amzn_linux_core.sh b/ci/docker/install/amzn_linux_core.sh deleted file mode 100755 index c13c969..0000000 --- a/ci/docker/install/amzn_linux_core.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -pushd . -yum install -y git -yum install -y wget -yum install -y sudo -yum install -y re2c -yum groupinstall -y 'Development Tools' - -# Ninja -git clone --recursive https://github.com/ninja-build/ninja.git -cd ninja -./configure.py --bootstrap -cp ninja /usr/local/bin -popd - -# CMake -pushd . -git clone --recursive https://github.com/Kitware/CMake.git --branch v3.10.2 -cd CMake -./bootstrap -make -j$(nproc) -make install -popd \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_julia.sh b/ci/docker/install/amzn_linux_julia.sh deleted file mode 100755 index bfaf3c4..0000000 --- a/ci/docker/install/amzn_linux_julia.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -wget -nv https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz -mv julia-0.5.0-linux-x86_64.tar.gz /tmp/ -tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz -rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz -# tar extracted in current directory -ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_library.sh b/ci/docker/install/amzn_linux_library.sh deleted file mode 100755 index 0470895..0000000 --- a/ci/docker/install/amzn_linux_library.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -yum -y install graphviz -pip install graphviz -pip install opencv-python \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_maven.sh b/ci/docker/install/amzn_linux_maven.sh deleted file mode 100755 index 22875d0..0000000 --- a/ci/docker/install/amzn_linux_maven.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -wget -nv http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -mv apache-maven-3.3.9-bin.tar.gz /tmp/ -tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz -yum install -y java-1.8.0-openjdk-devel \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_openblas.sh b/ci/docker/install/amzn_linux_openblas.sh deleted file mode 100755 index 94088d6..0000000 --- a/ci/docker/install/amzn_linux_openblas.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -pushd . -git clone https://github.com/xianyi/OpenBLAS -cd OpenBLAS -make FC=gfortran -j $(($(nproc) + 1)) -make PREFIX=/usr/local install -popd \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_opencv.sh b/ci/docker/install/amzn_linux_opencv.sh deleted file mode 100755 index 956407e..0000000 --- a/ci/docker/install/amzn_linux_opencv.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -pushd . -yum install -y python27 python27-setuptools -git clone https://github.com/opencv/opencv -cd opencv -mkdir -p build -cd build -cmake -DBUILD_opencv_gpu=OFF -DWITH_EIGEN=ON -DWITH_TBB=ON -DWITH_CUDA=OFF -DWITH_1394=OFF \ --DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local -GNinja .. -ninja install -popd \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_python2.sh b/ci/docker/install/amzn_linux_python2.sh deleted file mode 100755 index e099ad6..0000000 --- a/ci/docker/install/amzn_linux_python2.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -yum groupinstall -y "Development Tools" -yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip -ln -s -f /usr/bin/python2.7 /usr/bin/python2 -wget -nv https://bootstrap.pypa.io/get-pip.py -python2 get-pip.py -$(which easy_install-2.7) --upgrade pip -if [ -f /usr/local/bin/pip ] && [ -f /usr/bin/pip ]; then - mv /usr/bin/pip /usr/bin/pip.bak - ln /usr/local/bin/pip /usr/bin/pip -fi - -ln -s -f /usr/local/bin/pip /usr/bin/pip -for i in ipython[all] jupyter pandas scikit-image h5py pandas sklearn sympy; do echo "${i}..."; pip install -U $i >/dev/null; done diff --git a/ci/docker/install/amzn_linux_python3.sh b/ci/docker/install/amzn_linux_python3.sh deleted file mode 100755 index 3f80d7d..0000000 --- a/ci/docker/install/amzn_linux_python3.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -pushd . -wget -nv https://bootstrap.pypa.io/get-pip.py -mkdir py3 -cd py3 -wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz -tar -xvzf Python-3.5.2.tgz -cd Python-3.5.2 -yum install -y zlib-devel openssl-devel sqlite-devel bzip2-devel gdbm-devel ncurses-devel xz-devel readline-devel -./configure --prefix=/opt/ --with-zlib-dir=/usr/lib64 -make -j$(nproc) -mkdir /opt/bin -mkdir /opt/lib -make install -ln -s -f /opt/bin/python3 /usr/bin/python3 -cd ../.. -python3 get-pip.py -ln -s -f /opt/bin/pip /usr/bin/pip3 - -mkdir -p ~/.local/lib/python3.5/site-packages/ -pip3 install numpy -popd \ No newline at end of file diff --git a/ci/docker/install/amzn_linux_testdeps.sh b/ci/docker/install/amzn_linux_testdeps.sh deleted file mode 100755 index f5c49d9..0000000 --- a/ci/docker/install/amzn_linux_testdeps.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6' -pip3 install nose -ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3 -ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4 \ No newline at end of file diff --git a/ci/docker_cache.py b/ci/docker_cache.py new file mode 100755 index 0000000..7fdfbcf --- /dev/null +++ b/ci/docker_cache.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Utility to handle distributed docker cache. This is done by keeping the entire image chain of a docker container +on an S3 bucket. This utility allows cache creation and download. After execution, the cache will be in an identical +state as if the container would have been built locally already. +""" + +import os +import logging +import argparse +import sys +import boto3 +import tempfile +import pprint +import threading +import build as build_util +import botocore +import subprocess +from botocore.handlers import disable_signing +from subprocess import call, check_call, CalledProcessError +from joblib import Parallel, delayed + +S3_METADATA_IMAGE_ID_KEY = 'docker-image-id' +LOG_PROGRESS_PERCENTAGE_THRESHOLD = 10 + +cached_aws_session = None + + +class ProgressPercentage(object): + def __init__(self, object_name, size): + self._object_name = object_name + self._size = size + self._seen_so_far = 0 + self._last_percentage = 0 + self._lock = threading.Lock() + + def __call__(self, bytes_amount) -> None: + # To simplify we'll assume this is hooked up + # to a single filename. + with self._lock: + self._seen_so_far += bytes_amount + percentage = int((self._seen_so_far / self._size) * 100) + if (percentage - self._last_percentage) >= LOG_PROGRESS_PERCENTAGE_THRESHOLD: + self._last_percentage = percentage + logging.info('{}% of {}'.format(percentage, self._object_name)) + + +def build_save_containers(platforms, bucket) -> int: + """ + Entry point to build and upload all built dockerimages in parallel + :param platforms: List of platforms + :param bucket: S3 bucket name + :return: 1 if error occurred, 0 otherwise + """ + if len(platforms) == 0: + return 0 + + platform_results = Parallel(n_jobs=len(platforms), backend="multiprocessing")( + delayed(_build_save_container)(platform, bucket) + for platform in platforms) + + is_error = False + for platform_result in platform_results: + if platform_result is not None: + logging.error('Failed to generate {}'.format(platform_result)) + is_error = True + + return 1 if is_error else 0 + + +def _build_save_container(platform, bucket) -> str: + """ + Build image for passed platform and upload the cache to the specified S3 bucket + :param platform: Platform + :param bucket: Target s3 bucket + :return: Platform if failed, None otherwise + """ + docker_tag = build_util.get_docker_tag(platform) + + # Preload cache + # TODO: Allow to disable this in order to allow clean rebuilds + load_docker_cache(bucket_name=bucket, docker_tag=docker_tag) + + # Start building + logging.debug('Building {} as {}'.format(platform, docker_tag)) + try: + image_id = build_util.build_docker(docker_binary='docker', platform=platform) + logging.info('Built {} as {}'.format(docker_tag, image_id)) + + # Compile and upload tarfile + _compile_upload_cache_file(bucket_name=bucket, docker_tag=docker_tag, image_id=image_id) + return None + except Exception: + logging.exception('Unexpected exception during build of {}'.format(docker_tag)) + return platform + # Error handling is done by returning the errorous platform name. This is necessary due to + # Parallel being unable to handle exceptions + + +def _compile_upload_cache_file(bucket_name, docker_tag, image_id) -> None: + """ + Upload the passed image by id, tag it with docker tag and upload to S3 bucket + :param bucket_name: S3 bucket name + :param docker_tag: Docker tag + :param image_id: Image id + :return: None + """ + session = _get_aws_session() + s3_object = session.resource('s3').Object(bucket_name, docker_tag) + + remote_image_id = _get_remote_image_id(s3_object) + if remote_image_id == image_id: + logging.info('{} ({}) for {} has not been updated - skipping'.format(docker_tag, image_id, docker_tag)) + return + else: + logging.debug('Cached image {} differs from local {} for {}'.format(remote_image_id, image_id, docker_tag)) + + # Compile layers into tarfile + with tempfile.TemporaryDirectory() as temp_dir: + tar_file_path = _format_docker_cache_filepath(output_dir=temp_dir, docker_tag=docker_tag) + logging.debug('Writing layers of {} to {}'.format(docker_tag, tar_file_path)) + history_cmd = ['docker', 'history', '-q', docker_tag] + + image_ids_b = subprocess.check_output(history_cmd) + image_ids_str = image_ids_b.decode('utf-8').strip() + layer_ids = [id.strip() for id in image_ids_str.split('\n') if id != '<missing>'] + + # docker_tag is important to preserve the image name. Otherwise, the --cache-from feature will not be able to + # reference the loaded cache later on. The other layer ids are added to ensure all intermediary layers + # are preserved to allow resuming the cache at any point + cmd = ['docker', 'save', '-o', tar_file_path, docker_tag] + cmd.extend(layer_ids) + try: + check_call(cmd) + except CalledProcessError as e: + logging.error('Error during save of {} at {}. Command: {}'. + format(docker_tag, tar_file_path, pprint.pprint(cmd))) + return + + # Upload file + logging.info('Uploading {} to S3'.format(docker_tag)) + with open(tar_file_path, 'rb') as data: + s3_object.upload_fileobj( + Fileobj=data, + Callback=ProgressPercentage(object_name=docker_tag, size=os.path.getsize(tar_file_path)), + ExtraArgs={"Metadata": {S3_METADATA_IMAGE_ID_KEY: image_id}}) + logging.info('Uploaded {} to S3'.format(docker_tag)) + + +def _get_remote_image_id(s3_object) -> str: + """ + Get the image id of the docker cache which is represented by the S3 object + :param s3_object: S3 object + :return: Image id as string or None if object does not exist + """ + try: + if S3_METADATA_IMAGE_ID_KEY in s3_object.metadata: + cached_image_id = s3_object.metadata[S3_METADATA_IMAGE_ID_KEY] + return cached_image_id + else: + logging.debug('No cached image available for {}'.format(s3_object.key)) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "404": + logging.debug('{} does not exist in S3 yet'.format(s3_object.key)) + else: + raise + + return None + + +def load_docker_cache(bucket_name, docker_tag) -> None: + """ + Load the precompiled docker cache from the passed S3 bucket + :param bucket_name: S3 bucket name + :param docker_tag: Docker tag to load + :return: None + """ + # Allow anonymous access + s3_resource = boto3.resource('s3') + s3_resource.meta.client.meta.events.register('choose-signer.s3.*', disable_signing) + s3_object = s3_resource.Object(bucket_name, docker_tag) + + # Check if cache is still valid and exists + remote_image_id = _get_remote_image_id(s3_object) + if remote_image_id: + if _docker_layer_exists(remote_image_id): + logging.info('Local docker cache already present for {}'.format(docker_tag)) + return + else: + logging.info('Local docker cache not present for {}'.format(docker_tag)) + + # Download using public S3 endpoint (without requiring credentials) + with tempfile.TemporaryDirectory() as temp_dir: + tar_file_path = os.path.join(temp_dir, 'layers.tar') + s3_object.download_file( + Filename=tar_file_path, + Callback=ProgressPercentage(object_name=docker_tag, size=s3_object.content_length)) + + # Load layers + cmd = ['docker', 'load', '-i', tar_file_path] + try: + check_call(cmd) + logging.info('Docker cache for {} loaded successfully'.format(docker_tag)) + except CalledProcessError as e: + logging.error('Error during load of docker cache for {} at {}'.format(docker_tag, tar_file_path)) + logging.exception(e) + return + else: + logging.info('No cached remote image of {} present'.format(docker_tag)) + + +def _docker_layer_exists(layer_id) -> bool: + """ + Check if the docker cache contains the layer with the passed id + :param layer_id: layer id + :return: True if exists, False otherwise + """ + cmd = ['docker', 'images', '-q'] + image_ids_b = subprocess.check_output(cmd) + image_ids_str = image_ids_b.decode('utf-8').strip() + return layer_id in [id.strip() for id in image_ids_str.split('\n')] + + +def _get_aws_session() -> boto3.Session: # pragma: no cover + """ + Get the boto3 AWS session + :return: Session object + """ + global cached_aws_session + if cached_aws_session: + return cached_aws_session + + session = boto3.Session() # Uses IAM user credentials + cached_aws_session = session + return session + + +def _format_docker_cache_filepath(output_dir, docker_tag) -> str: + return os.path.join(output_dir, docker_tag.replace('/', '_') + '.tar') + + +def main() -> int: + # We need to be in the same directory than the script so the commands in the dockerfiles work as + # expected. But the script can be invoked from a different path + base = os.path.split(os.path.realpath(__file__))[0] + os.chdir(base) + + logging.getLogger().setLevel(logging.DEBUG) + logging.getLogger('botocore').setLevel(logging.INFO) + logging.getLogger('boto3').setLevel(logging.INFO) + logging.getLogger('urllib3').setLevel(logging.INFO) + logging.getLogger('s3transfer').setLevel(logging.INFO) + + def script_name() -> str: + return os.path.split(sys.argv[0])[1] + + logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name())) + + parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache",epilog="") + parser.add_argument("--docker-cache-bucket", + help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", + type=str, + required=True) + + args = parser.parse_args() + + platforms = build_util.get_platforms() + _get_aws_session() # Init AWS credentials + return build_save_containers(platforms=platforms, bucket=args.docker_cache_bucket) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ci/docker_cache_requirements b/ci/docker_cache_requirements new file mode 100644 index 0000000..47c16ff --- /dev/null +++ b/ci/docker_cache_requirements @@ -0,0 +1,8 @@ +boto3==1.7.13 +botocore==1.10.13 +docutils==0.14 +jmespath==0.9.3 +joblib==0.11 +python-dateutil==2.7.2 +s3transfer==0.1.13 +six==1.11.0 -- To stop receiving notification emails like this one, please contact marcoab...@apache.org.