pono closed pull request #8960: [Do not merge] Move to new CI URL: https://github.com/apache/incubator-mxnet/pull/8960
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7209b7c6a5..1c42e03958 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -150,3 +150,5 @@ List of Contributors * [Manu Seth](https://github.com/mseth10/) * [Calum Leslie](https://github.com/calumleslie) * [Andre Tamm](https://github.com/andretamm) +* [Marco de Abreu](https://github.com/marcoabreu) + - Marco is the creator of the current MXNet CI. diff --git a/Jenkinsfile b/Jenkinsfile index cbe63758ac..c4c16ad24f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,12 +52,12 @@ def init_git_win() { def make(docker_type, make_flag) { timeout(time: max_time, unit: 'MINUTES') { try { - sh "${docker_run} ${docker_type} make ${make_flag}" + sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}" } catch (exc) { echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch' - sh "${docker_run} ${docker_type} sudo make clean" - sh "${docker_run} ${docker_type} sudo make -C amalgamation/ clean" - sh "${docker_run} ${docker_type} make ${make_flag}" + sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean" + sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean" + sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}" } } } @@ -85,17 +85,17 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum // Python 2 def python2_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" - sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/unittest" - sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/train" + sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete" + sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/unittest" + sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/train" } } // Python 3 def python3_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" - sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" + sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete" + sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" } } @@ -120,7 +120,7 @@ def python3_gpu_ut(docker_type) { try { stage("Sanity Check") { timeout(time: max_time, unit: 'MINUTES') { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/sanity') { init_git() sh "python tools/license_header.py check" @@ -133,43 +133,82 @@ try { stage('Build') { parallel 'CPU: Openblas': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/build-cpu') { init_git() def flag = """ \ - DEV=1 \ - USE_PROFILER=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=openblas \ - -j\$(nproc) - """ + DEV=1 \ + USE_PROFILER=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + -j\$(nproc) + """ make("cpu", flag) pack_lib('cpu') } } }, - 'GPU: CUDA7.5+cuDNN5': { - node('mxnetlinux') { + 'CPU: MKLML': { + node('mxnetlinux-cpu') { + ws('workspace/build-mklml-cpu') { + init_git() + def flag = """ \ + DEV=1 \ + USE_PROFILER=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + USE_MKL2017=1 \ + USE_MKL2017_EXPERIMENTAL=1 \ + -j\$(nproc) + """ + make("cpu_mklml", flag) + pack_lib('mklml_cpu') + } + } + }, + 'GPU: MKLML': { + node('mxnetlinux-cpu') { + ws('workspace/build-mklml-gpu') { + init_git() + def flag = """ \ + DEV=1 \ + USE_PROFILER=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + USE_MKL2017=1 \ + USE_MKL2017_EXPERIMENTAL=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + -j\$(nproc) + """ + make("build_cuda", flag) + pack_lib('mklml_gpu') + } + } + }, + 'GPU: CUDA8.0+cuDNN5': { + node('mxnetlinux-cpu') { ws('workspace/build-gpu') { init_git() def flag = """ \ - DEV=1 \ - USE_PROFILER=1 \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - -j\$(nproc) - """ - make('gpu', flag) + DEV=1 \ + USE_PROFILER=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=1 \ + -j\$(nproc) + """ + make('build_cuda', flag) pack_lib('gpu') stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' } } }, 'Amalgamation MIN': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/amalgamationmin') { init_git() make('cpu', '-C amalgamation/ clean') @@ -178,7 +217,7 @@ try { } }, 'Amalgamation': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/amalgamation') { init_git() make('cpu', '-C amalgamation/ clean') @@ -186,93 +225,73 @@ try { } } }, - 'GPU: MKLML': { - node('mxnetlinux') { - ws('workspace/build-mklml') { - init_git() - def flag = """ \ - DEV=1 \ - USE_PROFILER=1 \ - USE_BLAS=openblas \ - USE_MKL2017=1 \ - USE_MKL2017_EXPERIMENTAL=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - -j\$(nproc) - """ - make('mklml_gpu', flag) - pack_lib('mklml') - } - } - }, - 'CPU windows':{ - node('mxnetwindows') { + 'Build CPU windows':{ + node('mxnetwindows-cpu') { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() bat """mkdir build_vc14_cpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_cpu - cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" + call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" + cd build_vc14_cpu + cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" bat 'C:\\mxnet\\build_vc14_cpu.bat' bat '''rmdir /s/q pkg_vc14_cpu - mkdir pkg_vc14_cpu\\lib - mkdir pkg_vc14_cpu\\python - mkdir pkg_vc14_cpu\\include - mkdir pkg_vc14_cpu\\build - copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib - copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build - xcopy python pkg_vc14_cpu\\python /E /I /Y - xcopy include pkg_vc14_cpu\\include /E /I /Y - xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y - xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y - xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ - ''' + mkdir pkg_vc14_cpu\\lib + mkdir pkg_vc14_cpu\\python + mkdir pkg_vc14_cpu\\include + mkdir pkg_vc14_cpu\\build + copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib + copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build + xcopy python pkg_vc14_cpu\\python /E /I /Y + xcopy include pkg_vc14_cpu\\include /E /I /Y + xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y + xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y + xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y + del /Q *.7z + 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ + ''' stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' } } } }, - 'GPU windows':{ - node('mxnetwindows') { + //Todo: Set specific CUDA_ARCh for windows builds in cmake + 'Build GPU windows':{ + node('mxnetwindows-cpu') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_gpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_gpu - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_gpu.bat' - bat '''rmdir /s/q pkg_vc14_gpu - mkdir pkg_vc14_gpu\\lib - mkdir pkg_vc14_gpu\\python - mkdir pkg_vc14_gpu\\include - mkdir pkg_vc14_gpu\\build - copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib - copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build - xcopy python pkg_vc14_gpu\\python /E /I /Y - xcopy include pkg_vc14_gpu\\include /E /I /Y - xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y - xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y - xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ - ''' - stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + init_git_win() + bat """mkdir build_vc14_gpu + call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" + cd build_vc14_gpu + cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" + bat 'C:\\mxnet\\build_vc14_gpu.bat' + bat '''rmdir /s/q pkg_vc14_gpu + mkdir pkg_vc14_gpu\\lib + mkdir pkg_vc14_gpu\\python + mkdir pkg_vc14_gpu\\include + mkdir pkg_vc14_gpu\\build + copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib + copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build + xcopy python pkg_vc14_gpu\\python /E /I /Y + xcopy include pkg_vc14_gpu\\include /E /I /Y + xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y + xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y + xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y + del /Q *.7z + 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ + ''' + stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' } } } - } + } } stage('Unit Test') { parallel 'Python2: CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-python2-cpu') { init_git() unpack_lib('cpu') @@ -281,7 +300,7 @@ try { } }, 'Python3: CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-python3-cpu') { init_git() unpack_lib('cpu') @@ -290,7 +309,7 @@ try { } }, 'Python2: GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-python2-gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -299,7 +318,7 @@ try { } }, 'Python3: GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-python3-gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -308,43 +327,43 @@ try { } }, 'Python2: MKLML-CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-python2-mklml-cpu') { init_git() - unpack_lib('mklml') - python2_ut('mklml_gpu') + unpack_lib('mklml_cpu') + python2_ut('cpu_mklml') } } }, 'Python2: MKLML-GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-python2-mklml-gpu') { init_git() - unpack_lib('mklml') - python2_gpu_ut('mklml_gpu') + unpack_lib('mklml_gpu') + python2_gpu_ut('gpu_mklml') } } }, 'Python3: MKLML-CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-python3-mklml-cpu') { init_git() - unpack_lib('mklml') - python3_ut('mklml_gpu') + unpack_lib('mklml_cpu') + python3_ut('cpu_mklml') } } }, 'Python3: MKLML-GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-python3-mklml-gpu') { init_git() - unpack_lib('mklml') - python3_gpu_ut('mklml_gpu') + unpack_lib('mklml_gpu') + python3_gpu_ut('gpu_mklml') } } }, 'Scala: CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-scala-cpu') { init_git() unpack_lib('cpu') @@ -356,7 +375,7 @@ try { } }, 'Perl: CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-perl-cpu') { init_git() unpack_lib('cpu') @@ -367,7 +386,7 @@ try { } }, 'Perl: GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-perl-gpu') { init_git() unpack_lib('gpu') @@ -378,7 +397,7 @@ try { } }, 'R: CPU': { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/ut-r-cpu') { init_git() unpack_lib('cpu') @@ -393,7 +412,7 @@ try { } }, 'R: GPU': { - node('mxnetlinux') { + node('mxnetlinux-gpu') { ws('workspace/ut-r-gpu') { init_git() unpack_lib('gpu') @@ -408,102 +427,102 @@ try { } }, 'Python 2: CPU Win':{ - node('mxnetwindows') { + node('mxnetwindows-cpu') { ws('workspace/ut-python-cpu') { init_git_win() unstash 'vc14_cpu' bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' + 7z x -y vc14_cpu.7z''' bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + xcopy C:\\mxnet\\model model /E /I /Y + call activate py2 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc + C:\\mxnet\\test_cpu.bat""" } } }, 'Python 3: CPU Win': { - node('mxnetwindows') { + node('mxnetwindows-cpu') { ws('workspace/ut-python-cpu') { init_git_win() unstash 'vc14_cpu' bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" - } + 7z x -y vc14_cpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py3 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc + C:\\mxnet\\test_cpu.bat""" + } } }, 'Python 2: GPU Win':{ - node('mxnetwindows') { + node('mxnetwindows-gpu') { ws('workspace/ut-python-gpu') { - init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + init_git_win() + unstash 'vc14_gpu' + bat '''rmdir /s/q pkg_vc14_gpu + 7z x -y vc14_gpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py2 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc + C:\\mxnet\\test_gpu.bat""" } } }, 'Python 3: GPU Win':{ - node('mxnetwindows') { + node('mxnetwindows-gpu') { ws('workspace/ut-python-gpu') { - init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + init_git_win() + unstash 'vc14_gpu' + bat '''rmdir /s/q pkg_vc14_gpu + 7z x -y vc14_gpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py3 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc + C:\\mxnet\\test_gpu.bat""" } } } } stage('Integration Test') { - parallel 'Python': { - node('mxnetlinux') { + parallel 'Python GPU': { + node('mxnetlinux-gpu') { ws('workspace/it-python-gpu') { init_git() unpack_lib('gpu') timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu PYTHONPATH=./python/ python example/image-classification/test_score.py" + sh "${docker_run} gpu --dockerbinary nvidia-docker PYTHONPATH=./python/ python example/image-classification/test_score.py" } } } }, - 'Caffe': { - node('mxnetlinux') { + 'Caffe GPU': { + node('mxnetlinux-gpu') { ws('workspace/it-caffe') { init_git() unpack_lib('gpu') timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} caffe_gpu PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py" + sh "${docker_run} caffe_gpu --dockerbinary nvidia-docker PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py" } } } }, - 'cpp-package': { - node('mxnetlinux') { + 'cpp-package GPU': { + node('mxnetlinux-gpu') { ws('workspace/it-cpp-package') { init_git() unpack_lib('gpu') unstash 'cpp_test_score' timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu cpp-package/tests/ci_test.sh" + sh "${docker_run} gpu --dockerbinary nvidia-docker cpp-package/tests/ci_test.sh" } } } @@ -511,7 +530,7 @@ try { } stage('Deploy') { - node('mxnetlinux') { + node('mxnetlinux-cpu') { ws('workspace/docs') { if (env.BRANCH_NAME == "master") { init_git() @@ -524,13 +543,13 @@ try { // set build status to success at the end currentBuild.result = "SUCCESS" } catch (caughtError) { - node("mxnetlinux") { + node("mxnetlinux-cpu") { sh "echo caught ${caughtError}" err = caughtError currentBuild.result = "FAILURE" } } finally { - node("mxnetlinux") { + node("mxnetlinux-cpu") { // Only send email if master failed if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") { emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' diff --git a/Makefile b/Makefile index 72dd26e0e4..f506257dee 100644 --- a/Makefile +++ b/Makefile @@ -267,6 +267,7 @@ ifeq ($(CUDA_ARCH),) CUDA_ARCH += $(shell $(NVCC) -cuda $(COMPRESS) --x cu /dev/null -o /dev/null >/dev/null 2>&1 && \ echo $(COMPRESS)) endif +$(info Running CUDA_ARCH: $(CUDA_ARCH)) endif # ps-lite @@ -330,6 +331,9 @@ ifeq ($(USE_CUDA), 1) CFLAGS += -I$(ROOTDIR)/3rdparty/cub ALL_DEP += $(CUOBJ) $(EXTRA_CUOBJ) $(PLUGIN_CUOBJ) LDFLAGS += -lcuda -lcufft -lnvrtc + # Make sure to add stubs as fallback in order to be able to build + # without full CUDA install (especially if run without nvidia-docker) + LDFLAGS += -L/usr/local/cuda/lib64/stubs SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu ifeq ($(USE_NCCL), 1) ifneq ($(USE_NCCL_PATH), NONE) diff --git a/perl-package/test.sh b/perl-package/test.sh index 1a4bd7227a..417e00a03c 100755 --- a/perl-package/test.sh +++ b/perl-package/test.sh @@ -29,4 +29,4 @@ make install || exit -1 cd ${MXNET_HOME}/perl-package/AI-MXNet/ perl Makefile.PL INSTALL_BASE=${MXNET_HOME}/perl5 -make test || exit -1 +make test TEST_VERBOSE=1 || exit -1 # Add debug output to test log diff --git a/tests/ci_build/Dockerfile.build_cuda b/tests/ci_build/Dockerfile.build_cuda new file mode 100644 index 0000000000..5fccec7681 --- /dev/null +++ b/tests/ci_build/Dockerfile.build_cuda @@ -0,0 +1,26 @@ +FROM nvidia/cuda:8.0-cudnn5-devel +# cuda8.0 has to be used because this is the first ubuntu16.04 container +# which is required due to OpenBLAS being incompatible with ubuntu14.04 +# the reason we used a gpu base container because we are going to test MKLDNN +# operator implementation against GPU implementation + +COPY install/ubuntu_install_core.sh /install/ +RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ +RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ +RUN /install/ubuntu_install_scala.sh +COPY install/ubuntu_install_r.sh /install/ +RUN /install/ubuntu_install_r.sh +COPY install/ubuntu_install_perl.sh /install/ +RUN /install/ubuntu_install_perl.sh + +# Allows to run tasks on a CPU without nvidia-docker and GPU +COPY install/ubuntu_install_nvidia.sh /install/ +RUN /install/ubuntu_install_nvidia.sh + +# Add MKLML libraries +RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz +RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_* + +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/tests/ci_build/Dockerfile.caffe_gpu b/tests/ci_build/Dockerfile.caffe_gpu index 4f6522dab8..34c4625bdc 100644 --- a/tests/ci_build/Dockerfile.caffe_gpu +++ b/tests/ci_build/Dockerfile.caffe_gpu @@ -1,4 +1,6 @@ -FROM nvidia/cuda:7.5-cudnn5-devel +FROM nvidia/cuda:8.0-cudnn5-devel +# cuda8.0 has to be used because this is the first ubuntu16.04 container +# which is required due to OpenBLAS being incompatible with ubuntu14.04 COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh @@ -18,6 +20,15 @@ RUN cd /; git clone http://github.com/BVLC/caffe.git; cd caffe; \ RUN echo "CPU_ONLY := 1" >> /caffe/Makefile.config +# Fixes https://github.com/BVLC/caffe/issues/5658 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide +RUN echo "INCLUDE_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/include/hdf5/serial/ " >> /caffe/Makefile.config +RUN echo "LIBRARY_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial " >> /caffe/Makefile.config + +# Fixes https://github.com/BVLC/caffe/issues/4333 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide +# Note: This is only valid on Ubuntu16.04 - the version numbers are bound to the distribution +RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5.so +RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so + RUN cd caffe; make all pycaffe -j$(nproc) RUN cd caffe/python; for req in $(cat requirements.txt); do pip2 install $req; done diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu index c7bb0af0f7..226054a597 100644 --- a/tests/ci_build/Dockerfile.cpu +++ b/tests/ci_build/Dockerfile.cpu @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:16.04 COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh diff --git a/tests/ci_build/Dockerfile.cpu_mklml b/tests/ci_build/Dockerfile.cpu_mklml new file mode 100644 index 0000000000..faa78646c5 --- /dev/null +++ b/tests/ci_build/Dockerfile.cpu_mklml @@ -0,0 +1,18 @@ +FROM ubuntu:16.04 + +COPY install/ubuntu_install_core.sh /install/ +RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ +RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ +RUN /install/ubuntu_install_scala.sh +COPY install/ubuntu_install_r.sh /install/ +RUN /install/ubuntu_install_r.sh +COPY install/ubuntu_install_perl.sh /install/ +RUN /install/ubuntu_install_perl.sh + +# Add MKLML library, compatiable with Ubuntu16.04 +RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz +RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_* + +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index a2893a9fb4..2483e62b99 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -1,4 +1,6 @@ -FROM nvidia/cuda:7.5-cudnn5-devel +FROM nvidia/cuda:8.0-cudnn5-devel +# cuda8.0 has to be used because this is the first ubuntu16.04 container +# which is required due to OpenBLAS being incompatible with ubuntu14.04 COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh diff --git a/tests/ci_build/Dockerfile.mklml_gpu b/tests/ci_build/Dockerfile.gpu_mklml similarity index 65% rename from tests/ci_build/Dockerfile.mklml_gpu rename to tests/ci_build/Dockerfile.gpu_mklml index 185681cb52..2c3564c45e 100644 --- a/tests/ci_build/Dockerfile.mklml_gpu +++ b/tests/ci_build/Dockerfile.gpu_mklml @@ -1,4 +1,6 @@ -FROM nvidia/cuda:7.5-cudnn5-devel +FROM nvidia/cuda:8.0-cudnn5-devel +# cuda8.0 has to be used because this is the first ubuntu16.04 container +# # which is required due to OpenBLAS being incompatible with ubuntu14.04 # the reason we used a gpu base container because we are going to test MKLDNN # operator implementation against GPU implementation @@ -9,7 +11,8 @@ RUN /install/ubuntu_install_python.sh COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh -RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.10/mklml_lnx_2018.0.20170908.tgz +# Add MKLML library, compatible with Ubuntu16.04 +RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_* ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/tests/ci_build/Dockerfile.lint b/tests/ci_build/Dockerfile.lint index b19b7676ec..a72b3f8886 100644 --- a/tests/ci_build/Dockerfile.lint +++ b/tests/ci_build/Dockerfile.lint @@ -1,5 +1,6 @@ # For lint test -FROM ubuntu:14.04 +FROM ubuntu:16.04 -RUN apt-get update && apt-get install -y python-pip +# Sudo is not present on ubuntu16.04 +RUN apt-get update && apt-get install -y python-pip sudo RUN pip install cpplint pylint diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh index 79fcd86a5d..512eb7a58c 100755 --- a/tests/ci_build/ci_build.sh +++ b/tests/ci_build/ci_build.sh @@ -55,6 +55,12 @@ if [[ "$1" == "-it" ]]; then shift 1 fi +if [[ "$1" == "--dockerbinary" ]]; then + DOCKER_BINARY="$2" + echo "Using custom Docker Engine: ${DOCKER_BINARY}" + shift 2 +fi + if [[ ! -f "${DOCKERFILE_PATH}" ]]; then echo "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\"" exit 1 @@ -73,11 +79,15 @@ if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then exit 1 fi -# Use nvidia-docker if the container is GPU. -if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then - DOCKER_BINARY="nvidia-docker" -else - DOCKER_BINARY="docker" +# Only set docker binary automatically if it has not been specified +if [[ -z "${DOCKER_BINARY}" ]]; then + # Use nvidia-docker if the container is GPU. + if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then + DOCKER_BINARY="nvidia-docker" + else + DOCKER_BINARY="docker" + fi + echo "Automatically assuming ${DOCKER_BINARY} as docker binary" fi # Helper function to traverse directories up until given file is found. @@ -147,6 +157,7 @@ ${DOCKER_BINARY} run --rm --pid=host \ -e "CI_BUILD_UID=$(id -u)" \ -e "CI_BUILD_GROUP=$(id -g -n)" \ -e "CI_BUILD_GID=$(id -g)" \ + -e "CUDA_ARCH=-gencode arch=compute_52,code=[sm_52,compute_52] --fatbin-options -compress-all" \ ${CI_DOCKER_EXTRA_PARAMS[@]} \ ${DOCKER_IMG_NAME} \ ${PRE_COMMAND} \ diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/tests/ci_build/install/ubuntu_install_core.sh index 49475740d2..eefd7590cd 100755 --- a/tests/ci_build/install/ubuntu_install_core.sh +++ b/tests/ci_build/install/ubuntu_install_core.sh @@ -21,6 +21,9 @@ apt-get update && apt-get install -y \ build-essential git libopenblas-dev liblapack-dev libopencv-dev \ - libcurl4-openssl-dev libgtest-dev cmake wget unzip + libcurl4-openssl-dev libgtest-dev cmake wget unzip sudo + +# Link Openblas to Cblas as this link does not exist on ubuntu16.04 +ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib diff --git a/tests/ci_build/install/ubuntu_install_nvidia.sh b/tests/ci_build/install/ubuntu_install_nvidia.sh new file mode 100755 index 0000000000..71fde8e800 --- /dev/null +++ b/tests/ci_build/install/ubuntu_install_nvidia.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# install nvidia libraries to compile and run CUDA without +# the necessity of nvidia-docker and a GPU + +# Needed to run add-apt-repository +apt update && apt install -y software-properties-common + +add-apt-repository -y ppa:graphics-drivers + +# Retrieve ppa:graphics-drivers and install nvidia-drivers. +# Note: DEBIAN_FRONTEND required to skip the interactive setup steps +apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y nvidia-384 diff --git a/tests/ci_build/pip_tests/Dockerfile.in.pip_cpu b/tests/ci_build/pip_tests/Dockerfile.in.pip_cpu index dfd675b890..de4629fab2 100644 --- a/tests/ci_build/pip_tests/Dockerfile.in.pip_cpu +++ b/tests/ci_build/pip_tests/Dockerfile.in.pip_cpu @@ -1,4 +1,4 @@ # -*- mode: dockerfile -*- # dockerfile to test pip installation on CPU -FROM ubuntu:14.04 +FROM ubuntu:16.04 diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 8ee4bfae00..e044df0705 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -19,6 +19,7 @@ import numpy as np from mxnet import gluon from mxnet.test_utils import assert_almost_equal, default_context +import unittest def test_loss_ndarray(): @@ -160,6 +161,7 @@ def test_l1_loss(): assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1 [email protected]("flaky test. https://github.com/apache/incubator-mxnet/issues/8892";) def test_ctc_loss(): loss = gluon.loss.CTCLoss() l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[1,0,-1,-1],[2,1,1,-1]])) @@ -185,7 +187,7 @@ def test_ctc_loss(): l = loss(mx.nd.ones((2,25,4)), mx.nd.array([[2,1,3,3],[3,2,2,3]]), mx.nd.array([20,20]), mx.nd.array([2,3])) mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) - [email protected]("flaky test. https://github.com/apache/incubator-mxnet/issues/8892";) def test_ctc_loss_train(): np.random.seed(1234) N = 20 ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
