This is an automated email from the ASF dual-hosted git repository. cjolivier01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new f950702 [MXNET-136] Enabling USE_DIST_KVSTORE flag for CI (#10232) f950702 is described below commit f95070239e3b2b1722cfc29f6a1807965b861cdf Author: Rahul Huilgol <rahulhuil...@gmail.com> AuthorDate: Wed Apr 4 14:55:39 2018 -0700 [MXNET-136] Enabling USE_DIST_KVSTORE flag for CI (#10232) * test dist_kvstore compilation for make * add cmake functions * install zmq * update ps-lite * fix scala build failure * updating ps-lite * stash more libs for scala link * update ps-lite executable path --- 3rdparty/ps-lite | 2 +- Jenkinsfile | 12 +++++++----- ci/docker/install/ubuntu_core.sh | 1 + ci/docker/runtime_functions.sh | 16 ++++++++++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite index aee3252..a6dda54 160000 --- a/3rdparty/ps-lite +++ b/3rdparty/ps-lite @@ -1 +1 @@ -Subproject commit aee325276bccb092f516df0bce30d3a8333f4038 +Subproject commit a6dda54604a07d1fb21b016ed1e3f4246b08222a diff --git a/Jenkinsfile b/Jenkinsfile index 45b86fb..3892906 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,6 +22,8 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a' +// for scala build, need to pass extra libs when run with dist_kvstore +mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' @@ -164,7 +166,7 @@ try { ws('workspace/build-cpu-openblas') { init_git() sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas" - pack_lib('cpu') + pack_lib('cpu', mx_dist_lib) } } }, @@ -224,8 +226,8 @@ try { node('mxnetlinux-cpu') { ws('workspace/build-gpu') { init_git() - sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7" - pack_lib('gpu') + sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7" + pack_lib('gpu', mx_dist_lib) stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer' } @@ -475,7 +477,7 @@ try { node('mxnetlinux-cpu') { ws('workspace/ut-scala-cpu') { init_git() - unpack_lib('cpu') + unpack_lib('cpu', mx_dist_lib) timeout(time: max_time, unit: 'MINUTES') { sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala" } @@ -486,7 +488,7 @@ try { node('mxnetlinux-gpu') { ws('workspace/ut-scala-gpu') { init_git() - unpack_lib('gpu') + unpack_lib('gpu', mx_dist_lib) timeout(time: max_time, unit: 'MINUTES') { sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_scala" } diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index dc9b091..f588d06 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -33,6 +33,7 @@ apt-get install -y \ wget \ unzip \ sudo \ + libzmq3-dev \ software-properties-common \ ninja-build diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 3d58f34..a97f6b9 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -131,6 +131,7 @@ build_amzn_linux_cpu() { -DCMAKE_BUILD_TYPE=RelWithDebInfo\ -DUSE_MKL_IF_AVAILABLE=OFF\ -DUSE_LAPACK=OFF\ + -DUSE_DIST_KVSTORE=ON\ -G Ninja /work/mxnet ninja export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so @@ -180,6 +181,7 @@ build_centos7_cpu() { USE_LAPACK=1 \ USE_LAPACK_PATH=/usr/lib64/liblapack.so \ USE_BLAS=openblas \ + USE_DIST_KVSTORE=1 \ -j$(nproc) } @@ -206,6 +208,7 @@ build_centos7_gpu() { USE_CUDA=1 \ USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ + USE_DIST_KVSTORE=1 \ -j$(nproc) } @@ -215,6 +218,7 @@ build_ubuntu_cpu_openblas() { DEV=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ + USE_DIST_KVSTORE=1 \ -j$(nproc) } @@ -224,6 +228,7 @@ build_ubuntu_cpu_clang39() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ + USE_DIST_KVSTORE=1 \ CXX=clang++-3.9 \ CC=clang-3.9 \ -j$(nproc) @@ -235,6 +240,7 @@ build_ubuntu_cpu_clang50() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=1 \ + USE_DIST_KVSTORE=1 \ CXX=clang++-5.0 \ CC=clang-5.0 \ -j$(nproc) @@ -296,6 +302,7 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ USE_CPP_PACKAGE=1 \ + USE_DIST_KVSTORE=1 \ -j$(nproc) } @@ -336,6 +343,7 @@ build_ubuntu_gpu_cmake() { -DUSE_CUDNN=1 \ -DUSE_MKLML_MKL=0 \ -DUSE_MKLDNN=0 \ + -DUSE_DIST_KVSTORE=1 \ -DCMAKE_BUILD_TYPE=Release \ -G Ninja \ /work/mxnet @@ -423,14 +431,14 @@ unittest_ubuntu_python3_quantization_gpu() { unittest_ubuntu_cpu_scala() { set -ex - make scalapkg USE_BLAS=openblas - make scalatest USE_BLAS=openblas + make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 + make scalatest USE_BLAS=openblas USE_DIST_KVSTORE=1 } unittest_ubuntu_gpu_scala() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 - make scalatest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 + make scalatest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 } unittest_ubuntu_cpugpu_perl() { -- To stop receiving notification emails like this one, please contact cjolivie...@apache.org.