This is an automated email from the ASF dual-hosted git repository.
zha0q1 pushed a commit to branch v1.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.x by this push:
new fc37c75 [FEATURE]Migrating all CD pipelines to Ninja build + fix
cu112 CD pipeline (#19974)
fc37c75 is described below
commit fc37c759fc12388b57f356e196ef06faa8e8e889
Author: Rohit Kumar Srivastava <[email protected]>
AuthorDate: Thu Mar 11 16:20:01 2021 -0800
[FEATURE]Migrating all CD pipelines to Ninja build + fix cu112 CD pipeline
(#19974)
* migrating cd builds to ninja + removing static links to nvidia libs and
leagacy cuda versions
* installing NCCL manually for cuda11.2 container
* set MSHADOW_USE_CUDNN=1 in CMakelists of mshadow to build properly for
CUDNN support
* adding coverage to cd requirements file to fix cu100, cu101 and cu102
tests
* updating cd_test containers to ubuntu 18
* adding cmake config for linux native and adding USE_KV_STORE in linux_cpu
* updating zmq builds to statically link to libmxnet.so
* updating toolchains for r, clang and llvm for ubuntu18. OpenBlas Static
link for 'distribution' build type only. Fix caffe build to use openCV 3.
Remove leagacy Clang 3.9 from CI
* fix versions for pip install in ubuntu_core_sh add new search path for
cuDNN
* finxing cudnn link problem for CUDA<=11.0
* adding library paths for libjpegturbo and lapack to fix failing CI on
ubuntu 18 images
* removing ASAN integration test from miscellaneous CI as its not required
* fix lapack path for gpu builds
* correctly installing libjpegturbo for ubuntu 18
* updating docker images of r,jekyll,julia etc test containers+ fix java
version to 8
* installing libomp.so
* removing debug test as its not required. Code clean-up
* adding alternate URL source for MNIST dataset as original website is down
* skipping flaky tests issue tracked #20011
Co-authored-by: Rohit Kumar Srivastava <[email protected]>
---
3rdparty/mshadow/CMakeLists.txt | 2 +-
CMakeLists.txt | 1 +
ci/docker/Dockerfile.build.ubuntu_build_cuda | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_c | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_jekyll | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_julia | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_lite | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_python | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_r | 4 +-
ci/docker/Dockerfile.build.ubuntu_cpu_scala | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu_cu100 | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu_cu101 | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu_cu102 | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu_cu110 | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu_cu112 | 4 +-
ci/docker/install/requirements | 1 +
ci/docker/install/ubuntu_caffe.sh | 1 +
ci/docker/install/ubuntu_clang.sh | 6 +-
ci/docker/install/ubuntu_core.sh | 10 +-
ci/docker/install/ubuntu_llvm.sh | 4 +-
ci/docker/install/ubuntu_onnx.sh | 2 +-
ci/docker/install/ubuntu_r.sh | 2 +-
ci/docker/runtime_functions.sh | 11 ++
ci/jenkins/Jenkinsfile_clang | 2 -
ci/jenkins/Jenkinsfile_miscellaneous | 6 +-
ci/jenkins/Jenkinsfile_unix_cpu | 1 -
cmake/ChooseBlas.cmake | 44 +++++
config/distribution/linux_cpu.cmake | 1 +
config/distribution/linux_cu100.cmake | 4 +-
config/distribution/linux_cu101.cmake | 4 +-
config/distribution/linux_cu102.cmake | 4 +-
config/distribution/linux_cu110.cmake | 4 +-
config/distribution/linux_cu112.cmake | 4 +-
.../{linux_cpu.cmake => linux_native.cmake} | 2 +-
cpp-package/example/get_data.sh | 6 +-
cpp-package/tests/ci_test.sh | 5 +-
tests/python/gpu/test_operator_gpu.py | 1 +
tools/dependencies/openblas.sh | 2 -
tools/dependencies/zmq.sh | 17 +-
tools/setup_gpu_build_tools.sh | 180 +++------------------
41 files changed, 165 insertions(+), 218 deletions(-)
diff --git a/3rdparty/mshadow/CMakeLists.txt b/3rdparty/mshadow/CMakeLists.txt
index 3b898a4..0fe4eec 100644
--- a/3rdparty/mshadow/CMakeLists.txt
+++ b/3rdparty/mshadow/CMakeLists.txt
@@ -42,7 +42,7 @@ else()
target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE=0)
endif()
if(USE_CUDNN)
- target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN)
+ target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN=1)
endif()
if(MSHADOW_IN_CXX11)
target_compile_definitions(mshadow INTERFACE MSHADOW_IN_CXX11)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ea49798..a0f153a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -148,6 +148,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND UNIX AND
NOT APPLE)
# Enforce DT_PATH instead of DT_RUNPATH
set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags")
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+ set(Protobuf_USE_STATIC_LIBS ON)
endif()
set(CMAKE_MODULE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda
b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 4840d37..7fa89f1 100644
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -16,12 +16,12 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build MXNet on Ubuntu 16.04 for GPU but on
+# Dockerfile to build MXNet on Ubuntu 18.04 for GPU but on
# a CPU-only instance. This restriction is caused by the CPP-
# package generation, requiring the actual CUDA library to be
# present
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu
b/ci/docker/Dockerfile.build.ubuntu_cpu
index 6893499..c8f23ed 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_c
b/ci/docker/Dockerfile.build.ubuntu_cpu_c
index c7969da..afe9eea 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_c
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_c
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll
b/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll
index bc91286..080039d 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_julia
b/ci/docker/Dockerfile.build.ubuntu_cpu_julia
index 6893499..c8f23ed 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_julia
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_julia
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_lite
b/ci/docker/Dockerfile.build.ubuntu_cpu_lite
index ca5618a..ff97e68 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_lite
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_lite
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_python
b/ci/docker/Dockerfile.build.ubuntu_cpu_python
index 6b217d4..74eae60 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_python
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_python
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_r
b/ci/docker/Dockerfile.build.ubuntu_cpu_r
index f41b651..3d17e09 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_r
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_r
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_scala
b/ci/docker/Dockerfile.build.ubuntu_cpu_scala
index 38874d2..4a5ffe2 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu_scala
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu_scala
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
index b792bae..c10e76e 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
-FROM nvidia/cuda:10.0-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu101
b/ci/docker/Dockerfile.build.ubuntu_gpu_cu101
index a3e0ece..ef07948 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu101
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu101
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
index 6c2582d..845a58d 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
-FROM nvidia/cuda:10.2-devel-ubuntu16.04
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
b/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
index db1f606..db5138b 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
-FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
b/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
index a6e479f..7fe0bce 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
-FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04
WORKDIR /work/deps
diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements
index ce94be7..1abdc00 100644
--- a/ci/docker/install/requirements
+++ b/ci/docker/install/requirements
@@ -32,3 +32,4 @@ astroid==2.3.3 # pylint and astroid need to be aligned
requests<2.19.0,>=2.18.4
scipy==1.2.1
setuptools
+coverage
diff --git a/ci/docker/install/ubuntu_caffe.sh
b/ci/docker/install/ubuntu_caffe.sh
index bda1c0b..503b57f 100755
--- a/ci/docker/install/ubuntu_caffe.sh
+++ b/ci/docker/install/ubuntu_caffe.sh
@@ -40,6 +40,7 @@ git clone http://github.com/BVLC/caffe.git
cd caffe
cp Makefile.config.example Makefile.config
+echo "OPENCV_VERSION := 3" >> Makefile.config
echo "CPU_ONLY := 1" >> Makefile.config
diff --git a/ci/docker/install/ubuntu_clang.sh
b/ci/docker/install/ubuntu_clang.sh
index ac1bdac..2788395 100755
--- a/ci/docker/install/ubuntu_clang.sh
+++ b/ci/docker/install/ubuntu_clang.sh
@@ -25,11 +25,9 @@ set -ex
apt-get update || true
# Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major
release)
wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
- apt-add-repository "deb http://apt.llvm.org/xenial/
llvm-toolchain-xenial-3.9 main" && \
- apt-add-repository "deb http://apt.llvm.org/xenial/
llvm-toolchain-xenial-6.0 main" && \
+ apt-add-repository "deb http://apt.llvm.org/bionic/
llvm-toolchain-bionic-6.0 main" && \
apt-get update && \
- apt-get install -y clang-3.9 clang-6.0 clang-tidy-6.0 && \
- clang-3.9 --version && \
+ apt-get install -y clang-6.0 clang-tidy-6.0 && \
clang-6.0 --version
# Use llvm's master version of run-clang-tidy.py. This version has mostly
minor updates, but
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index 16bcab8..53b5d4a 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -38,10 +38,14 @@ apt-get install -y \
libcurl4-openssl-dev \
libjemalloc-dev \
libhdf5-dev \
+ libomp5 \
+ libomp-dev \
liblapack-dev \
libopenblas-dev \
libopencv-dev \
- libturbojpeg \
+ libjpeg-turbo8-dev \
+ libjpeg8-dev \
+ libturbojpeg0-dev \
libzmq3-dev \
libtinfo-dev \
zlib1g-dev \
@@ -52,12 +56,12 @@ apt-get install -y \
sudo \
unzip \
vim-nox \
- default-jdk \
+ openjdk-8-jdk \
+ openjdk-8-jre \
wget
# Use libturbojpeg package as it is correctly compiled with -fPIC flag
# https://github.com/HaxeFoundation/hashlink/issues/147
-ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0
/usr/lib/x86_64-linux-gnu/libturbojpeg.so
# CMake 3.13.2+ is required
diff --git a/ci/docker/install/ubuntu_llvm.sh b/ci/docker/install/ubuntu_llvm.sh
index 8b6e765..476b3a2 100755
--- a/ci/docker/install/ubuntu_llvm.sh
+++ b/ci/docker/install/ubuntu_llvm.sh
@@ -17,9 +17,9 @@
# specific language governing permissions and limitations
# under the License.
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh
index 096a339..81c8755 100755
--- a/ci/docker/install/ubuntu_onnx.sh
+++ b/ci/docker/install/ubuntu_onnx.sh
@@ -30,4 +30,4 @@ echo "Installing libprotobuf-dev and protobuf-compiler ..."
apt-get update || true
apt-get install -y libprotobuf-dev protobuf-compiler
-pip3 install pytest pytest-cov pytest-xdist protobuf==3.5.2 onnx==1.7.0
Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 'numpy>1.16.0,<1.19.0'
gluonnlp gluoncv
+pip3 install pytest==6.2.2 pytest-cov==2.11.1 pytest-xdist==2.2.1
protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0
'numpy>1.16.0,<1.19.0' gluonnlp==0.10.0 gluoncv==0.8.0
diff --git a/ci/docker/install/ubuntu_r.sh b/ci/docker/install/ubuntu_r.sh
index 6105da8..44c0357 100755
--- a/ci/docker/install/ubuntu_r.sh
+++ b/ci/docker/install/ubuntu_r.sh
@@ -27,7 +27,7 @@
set -ex
cd "$(dirname "$0")"
# install libraries for mxnet's r package on ubuntu
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >>
/etc/apt/sources.list
+echo "deb http://cran.rstudio.com/bin/linux/ubuntu bionic-cran40/" >>
/etc/apt/sources.list
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys
E298A3A825C0D65DFD57CBB651716619E084DAB9
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 6043108..ceed020 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -489,6 +489,8 @@ build_ubuntu_cpu_openblas() {
USE_DIST_KVSTORE=1 \
USE_LIBJPEG_TURBO=1 \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+ USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
make cython PYTHON=python3
}
@@ -507,6 +509,7 @@ build_ubuntu_cpu_mkl() {
USE_INTEL_PATH=/opt/intel \
USE_DIST_KVSTORE=1 \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -681,6 +684,7 @@ build_ubuntu_cpu_mkldnn() {
USE_TVM_OP=1 \
USE_BLAS=openblas \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -696,6 +700,7 @@ build_ubuntu_cpu_mkldnn_mkl() {
USE_BLAS=mkl \
USE_SIGNAL_HANDLER=1 \
USE_INTEL_PATH=/opt/intel/ \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -744,6 +749,7 @@ build_ubuntu_gpu_mkldnn() {
USE_TVM_OP=0 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -761,6 +767,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
USE_TVM_OP=0 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -778,6 +785,7 @@ build_ubuntu_gpu_cuda101_cudnn7() {
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
make cython PYTHON=python3
}
@@ -1649,6 +1657,8 @@ build_ubuntu_cpu_docs() {
USE_DIST_KVSTORE=1 \
USE_LIBJPEG_TURBO=1 \
USE_SIGNAL_HANDLER=1 \
+ USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+ USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
-j$(nproc)
}
@@ -1974,6 +1984,7 @@ build_static_libmxnet() {
set -ex
pushd .
local mxnet_variant=${1:?"This function requires a python command as the
first argument"}
+ CMAKE_STATICBUILD=1
source tools/staticbuild/build.sh ${mxnet_variant}
popd
}
diff --git a/ci/jenkins/Jenkinsfile_clang b/ci/jenkins/Jenkinsfile_clang
index 029c720..8632032 100644
--- a/ci/jenkins/Jenkinsfile_clang
+++ b/ci/jenkins/Jenkinsfile_clang
@@ -34,10 +34,8 @@ utils.assign_node_labels(utility: 'utility', linux_cpu:
'mxnetlinux-cpu', linux_
utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
- custom_steps.compile_unix_clang_3_9_cpu(),
custom_steps.compile_unix_clang_6_cpu(),
custom_steps.compile_unix_clang_tidy_cpu(),
- custom_steps.compile_unix_clang_3_9_mkldnn_cpu(),
custom_steps.compile_unix_clang_6_mkldnn_cpu()
])
}
diff --git a/ci/jenkins/Jenkinsfile_miscellaneous
b/ci/jenkins/Jenkinsfile_miscellaneous
index dbf2a9e..fa0f390 100644
--- a/ci/jenkins/Jenkinsfile_miscellaneous
+++ b/ci/jenkins/Jenkinsfile_miscellaneous
@@ -38,11 +38,7 @@ core_logic: {
custom_steps.compile_unix_asan_cpu(),
custom_steps.compile_unix_amalgamation_min(),
custom_steps.compile_unix_amalgamation()
- ])
-
- utils.parallel_stage('Tests', [
- custom_steps.misc_asan_cpu()
- ])
+ ])
}
,
failure_handler: {
diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
index 71917de..2fa66c1 100644
--- a/ci/jenkins/Jenkinsfile_unix_cpu
+++ b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -45,7 +45,6 @@ core_logic: {
utils.parallel_stage('Tests', [
custom_steps.test_unix_python3_cpu(),
- custom_steps.test_unix_python3_debug_cpu(),
custom_steps.test_unix_python3_mkl_cpu(),
custom_steps.test_unix_python3_mkldnn_cpu(),
custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index e165947..3848a3c 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -45,6 +45,50 @@ elseif(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
add_definitions(-DMSHADOW_USE_CBLAS=1)
add_definitions(-DMSHADOW_USE_MKL=0)
add_definitions(-DMXNET_USE_BLAS_OPEN=1)
+ if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL "Distribution")
+ # check if we need to link to omp
+ execute_process(COMMAND ${CMAKE_NM} -g ${OpenBLAS_LIB}
+ COMMAND grep omp_get_num_threads
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ OUTPUT_VARIABLE OPENBLAS_USES_OMP_OUT
+ RESULT_VARIABLE OPENBLAS_USES_OMP_RET)
+ if(NOT OPENBLAS_USES_OMP_OUT STREQUAL "" AND NOT OPENBLAS_USES_OMP_RET AND
NOT USE_OPENMP)
+ message("Openblas uses OMP, automatically linking to it")
+ find_package(OpenMP REQUIRED)
+ message("OpenMP_CXX_LIBRARIES is ${OpenMP_CXX_LIBRARIES}")
+ list(APPEND mshadow_LINKER_LIBS "${OpenMP_CXX_LIBRARIES}")
+ endif()
+ # check if we need to link to gfortran
+ execute_process(COMMAND ${CMAKE_NM} -g ${OpenBLAS_LIB}
+ COMMAND grep gfortran
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ OUTPUT_VARIABLE OPENBLAS_USES_GFORTRAN_OUT
+ RESULT_VARIABLE OPENBLAS_USES_GFORTRAN_RET)
+ if(NOT OPENBLAS_USES_GFORTRAN_OUT STREQUAL "" AND NOT
OPENBLAS_USES_GFORTRAN_RET)
+ message("Openblas uses GFortran, automatically linking to it")
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/temp/CMakeLists.txt"
+ "cmake_minimum_required(VERSION ${CMAKE_VERSION})
+project(CheckFortran Fortran)
+set(CMAKE_Fortran_COMPILER gfortran)
+file(WRITE \"${CMAKE_CURRENT_BINARY_DIR}/temp/FortranDir.cmake\"
+\"
+set(FORTRAN_DIR \\\"\$\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\}\\\")
+\")
+")
+ execute_process(
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/temp/
+ COMMAND ${CMAKE_COMMAND} .
+ )
+ set(FORTRAN_DIR "")
+ include(build/temp/FortranDir.cmake)
+ find_library(FORTRAN_LIB NAMES gfortran HINTS ${FORTRAN_DIR})
+ message("FORTRAN_DIR is ${FORTRAN_DIR}")
+ message("FORTRAN_LIB is ${FORTRAN_LIB}")
+ list(APPEND mshadow_LINKER_LIBS ${FORTRAN_LIB})
+ file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/temp/")
+ endif()
+ endif()
+
elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
find_package(MKL REQUIRED)
include_directories(SYSTEM ${MKL_INCLUDE_DIR})
diff --git a/config/distribution/linux_cpu.cmake
b/config/distribution/linux_cpu.cmake
index cad3485..5a8673b 100644
--- a/config/distribution/linux_cpu.cmake
+++ b/config/distribution/linux_cpu.cmake
@@ -28,3 +28,4 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
diff --git a/config/distribution/linux_cu100.cmake
b/config/distribution/linux_cu100.cmake
index d26b4d7..e4b0095 100644
--- a/config/distribution/linux_cu100.cmake
+++ b/config/distribution/linux_cu100.cmake
@@ -21,6 +21,7 @@ set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
set(USE_CUDA ON CACHE BOOL "Build with CUDA support")
set(USE_CUDNN ON CACHE BOOL "Build with CUDA support")
+set(USE_NCCL ON CACHE BOOL "Build with NCCL support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
@@ -29,6 +30,7 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
-
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
set(CUDACXX "/usr/local/cuda-10.0/bin/nvcc" CACHE STRING "Cuda compiler")
set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0;7.5" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu101.cmake
b/config/distribution/linux_cu101.cmake
index aaf76cc..c6336e2 100644
--- a/config/distribution/linux_cu101.cmake
+++ b/config/distribution/linux_cu101.cmake
@@ -23,6 +23,7 @@ set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
set(USE_CUDA ON CACHE BOOL "Build with CUDA support")
set(USE_CUDNN ON CACHE BOOL "Build with CUDA support")
+set(USE_NCCL ON CACHE BOOL "Build with NCCL support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
@@ -31,6 +32,7 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
-
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
set(CUDACXX "/usr/local/cuda-10.1/bin/nvcc" CACHE STRING "Cuda compiler")
set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0;7.5" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu102.cmake
b/config/distribution/linux_cu102.cmake
index 6b57568..a405a50 100644
--- a/config/distribution/linux_cu102.cmake
+++ b/config/distribution/linux_cu102.cmake
@@ -21,6 +21,7 @@ set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
set(USE_CUDA ON CACHE BOOL "Build with CUDA support")
set(USE_CUDNN ON CACHE BOOL "Build with CUDA support")
+set(USE_NCCL ON CACHE BOOL "Build with NCCL support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
@@ -29,6 +30,7 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
-
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
set(CUDACXX "/usr/local/cuda-10.2/bin/nvcc" CACHE STRING "Cuda compiler")
set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0;7.5" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu110.cmake
b/config/distribution/linux_cu110.cmake
index 7d44a99..c58ec9d 100644
--- a/config/distribution/linux_cu110.cmake
+++ b/config/distribution/linux_cu110.cmake
@@ -21,6 +21,7 @@ set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
set(USE_CUDA ON CACHE BOOL "Build with CUDA support")
set(USE_CUDNN ON CACHE BOOL "Build with CUDA support")
+set(USE_NCCL ON CACHE BOOL "Build with NCCL support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
@@ -29,6 +30,7 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
-
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
set(CUDACXX "/usr/local/cuda-11.0/bin/nvcc" CACHE STRING "Cuda compiler")
set(MXNET_CUDA_ARCH "5.0;6.0;7.0;8.0" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cu112.cmake
b/config/distribution/linux_cu112.cmake
index 6c9a876..b0d3d86 100644
--- a/config/distribution/linux_cu112.cmake
+++ b/config/distribution/linux_cu112.cmake
@@ -21,6 +21,7 @@ set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
set(USE_CUDA ON CACHE BOOL "Build with CUDA support")
set(USE_CUDNN ON CACHE BOOL "Build with CUDA support")
+set(USE_NCCL ON CACHE BOOL "Build with NCCL support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
@@ -29,6 +30,7 @@ set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
-
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
set(CUDACXX "/usr/local/cuda-11.2/bin/nvcc" CACHE STRING "Cuda compiler")
set(MXNET_CUDA_ARCH "5.0;6.0;7.0;8.0;8.6" CACHE STRING "Cuda architectures")
diff --git a/config/distribution/linux_cpu.cmake
b/config/distribution/linux_native.cmake
similarity index 96%
copy from config/distribution/linux_cpu.cmake
copy to config/distribution/linux_native.cmake
index cad3485..5673f19 100644
--- a/config/distribution/linux_cpu.cmake
+++ b/config/distribution/linux_native.cmake
@@ -23,7 +23,7 @@ set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
-set(USE_MKLDNN ON CACHE BOOL "Build with MKL-DNN support")
+set(USE_MKLDNN OFF CACHE BOOL "Build with MKL-DNN support")
set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
diff --git a/cpp-package/example/get_data.sh b/cpp-package/example/get_data.sh
index e110772..77677c3 100755
--- a/cpp-package/example/get_data.sh
+++ b/cpp-package/example/get_data.sh
@@ -60,5 +60,9 @@ FILES=(
"http://data.mxnet.io/data/mnist_train.csv.gz";)
for FILE in ${FILES[@]}; do
- download ${FILE}
+ if curl --output /dev/null --silent --head --fail "$FILE"; then
+ download ${FILE}
+ else
+ download "https://web.archive.org/web/20160828233817/$FILE";
+ fi
done
diff --git a/cpp-package/tests/ci_test.sh b/cpp-package/tests/ci_test.sh
index 39f9e06..58f04b3 100755
--- a/cpp-package/tests/ci_test.sh
+++ b/cpp-package/tests/ci_test.sh
@@ -60,8 +60,9 @@ cp ../../build/cpp-package/example/test_score .
cp ../../build/cpp-package/example/test_ndarray_copy .
./test_ndarray_copy
-cp ../../build/cpp-package/example/test_regress_label .
-./test_regress_label
+# skippping temporarily, tracked by
https://github.com/apache/incubator-mxnet/issues/20011
+#cp ../../build/cpp-package/example/test_regress_label .
+#./test_regress_label
sh unittests/unit_test_mlp_csv.sh
diff --git a/tests/python/gpu/test_operator_gpu.py
b/tests/python/gpu/test_operator_gpu.py
index 5fee473..ceb8c6e 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -2248,6 +2248,7 @@ def kernel_error_check_symbolic():
f.forward()
g = f.outputs[0].asnumpy()
[email protected]('skippping temporarily, tracked by
https://github.com/apache/incubator-mxnet/issues/20011')
def test_kernel_error_checking():
# Running tests that may throw exceptions out of worker threads will stop
CI testing
# if not run in a separate process (with its own address space for CUDA
compatibility).
diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
index cdc63b6..8871d04 100755
--- a/tools/dependencies/openblas.sh
+++ b/tools/dependencies/openblas.sh
@@ -42,8 +42,6 @@ if [[ ((! -e $DEPS_PATH/lib/libopenblas.a) && -z
"$CMAKE_STATICBUILD") ||
fi
$MAKE PREFIX=$DEPS_PATH install
-
-
if [[ -z "$CMAKE_STATICBUILD" ]]; then
# Manually removing .so to avoid linking against it
rm $DEPS_PATH/lib/libopenblasp-r${OPENBLAS_VERSION}.so
diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
index 11d7063..a0cba2e 100755
--- a/tools/dependencies/zmq.sh
+++ b/tools/dependencies/zmq.sh
@@ -20,6 +20,12 @@
# This script builds the static library of zeroMQ that can be used as
dependency of mxnet.
set -ex
ZEROMQ_VERSION=4.2.2
+if [[ $PLATFORM == 'darwin' ]]; then
+ DY_EXT="dylib"
+else
+ DY_EXT="so"
+fi
+
if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
# Download and build zmq
>&2 echo "Building zmq..."
@@ -37,5 +43,14 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
-D BUILD_SHARED_LIBS=OFF ..
$MAKE
$MAKE install
+
+ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
+ rm $DEPS_PATH/lib64/*zmq*$DY_EXT*
+ mkdir -p $DEPS_PATH/lib
+ cp $DEPS_PATH/lib64/*zmq* $DEPS_PATH/lib
+ else
+ rm $DEPS_PATH/lib/*zmq*$DY_EXT*
+ fi
+
popd
-fi
+fi
\ No newline at end of file
diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh
index fa4bc82..5a7528c 100755
--- a/tools/setup_gpu_build_tools.sh
+++ b/tools/setup_gpu_build_tools.sh
@@ -30,14 +30,13 @@ DEPS_PATH=$2
>&2 echo "Setting CUDA versions for $VARIANT"
if [[ $VARIANT == cu112* ]]; then
- CUDA_VERSION='11.2.135-1'
+ CUDA_VERSION='11.2.67-1'
CUDA_PATCH_VERSION='11.4.1.1026-1'
CUDA_LIBS_VERSION='10.2.3.135-1'
CUDA_SOLVER_VERSION='11.1.0.135-1'
- CUDA_NVTX_VERSION='11.2.67-1'
LIBCUDA_VERSION='460.32.03-0ubuntu1'
LIBCUDNN_VERSION='8.1.0.77-1+cuda11.2'
- LIBNCCL_VERSION='2.8.4-1+cuda11.2'
+ LIBNCCL_VERSION='2.8.3-1+cuda11.2'
LIBCUDART_VERSION='11.2.72-1'
LIBCUFFT_VERSION='10.4.0.135-1'
elif [[ $VARIANT == cu110* ]]; then
@@ -67,36 +66,6 @@ elif [[ $VARIANT == cu100* ]]; then
LIBCUDA_VERSION='410.48-0ubuntu1'
LIBCUDNN_VERSION='7.6.5.32-1+cuda10.0'
LIBNCCL_VERSION='2.5.6-1+cuda10.0'
-elif [[ $VARIANT == cu92* ]]; then
- CUDA_VERSION='9.2.148-1'
- CUDA_PATCH_VERSION='9.2.148.1-1'
- LIBCUDA_VERSION='396.44-0ubuntu1'
- LIBCUDNN_VERSION='7.6.5.32-1+cuda9.2'
- LIBNCCL_VERSION='2.4.8-1+cuda9.2'
-elif [[ $VARIANT == cu91* ]]; then
- CUDA_VERSION='9.1.85-1'
- CUDA_PATCH_VERSION='9.1.85.3-1'
- LIBCUDA_VERSION='396.44-0ubuntu1'
- LIBCUDNN_VERSION='7.1.3.16-1+cuda9.1'
- LIBNCCL_VERSION='2.2.12-1+cuda9.1'
-elif [[ $VARIANT == cu90* ]]; then
- CUDA_VERSION='9.0.176-1'
- CUDA_PATCH_VERSION='9.0.176.3-1'
- LIBCUDA_VERSION='384.145-0ubuntu1'
- LIBCUDNN_VERSION='7.6.5.32-1+cuda9.0'
- LIBNCCL_VERSION='2.5.6-1+cuda9.0'
-elif [[ $VARIANT == cu80* ]]; then
- CUDA_VERSION='8.0.61-1'
- CUDA_PATCH_VERSION='8.0.61.2-1'
- LIBCUDA_VERSION='375.88-0ubuntu1'
- LIBCUDNN_VERSION='7.2.1.38-1+cuda8.0'
- LIBNCCL_VERSION='2.3.4-1+cuda8.0'
-elif [[ $VARIANT == cu75* ]]; then
- CUDA_VERSION='7.5-18'
- CUDA_PATCH_VERSION='7.5-18'
- LIBCUDA_VERSION='375.88-0ubuntu1'
- LIBCUDNN_VERSION='6.0.21-1+cuda7.5'
- LIBNCCL_VERSION=''
fi
if [[ $VARIANT == cu* ]]; then
CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | tr '-' '.' | cut -d. -f1,2)
@@ -108,7 +77,7 @@ if [[ $VARIANT == cu* ]]; then
os_name=$(cat /etc/*release | grep '^ID=' | sed 's/^.*=//g')
os_version=$(cat /etc/*release | grep VERSION_ID | sed
's/^.*"\([0-9]*\)\.\([0-9]*\)"/\1\2/g')
os_id="${os_name}${os_version}"
- if [[ $CUDA_MAJOR_DASH == 9-* ]] || [[ $CUDA_MAJOR_DASH == 10-* ]] || [[
$CUDA_MAJOR_DASH == 11-* ]]; then
+ if [[ $CUDA_MAJOR_DASH == 9-* ]] || [[ $CUDA_MAJOR_DASH == 10-* ]] || [[
$CUDA_MAJOR_DASH == 11-* ]] ; then
os_id="ubuntu1604"
fi
export
PATH=/usr/lib/binutils-2.26/bin/:${PATH}:$DEPS_PATH/usr/local/cuda-$CUDA_MAJOR_VERSION/bin
@@ -135,14 +104,16 @@ if [[ $VARIANT == cu112* ]]; then
"libcusolver-${CUDA_MAJOR_DASH}_${CUDA_SOLVER_VERSION}_amd64.deb" \
"libcusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_SOLVER_VERSION}_amd64.deb" \
"cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_NVTX_VERSION}_amd64.deb" \
+ "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
"libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
"cuda-nvprof-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
"nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}_${LIBCUDNN_VERSION}_amd64.deb" \
)
ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
"libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
+ "libnccl2_${LIBNCCL_VERSION}_amd64.deb" \
)
elif [[ $VARIANT == cu110* ]]; then
cuda_files=( \
@@ -166,7 +137,9 @@ elif [[ $VARIANT == cu110* ]]; then
)
ml_files=( \
"libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}_${LIBCUDNN_VERSION}_amd64.deb" \
"libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
+ "libnccl2_${LIBNCCL_VERSION}_amd64.deb" \
)
elif [[ $VARIANT == cu102* ]]; then
cuda_files=( \
@@ -191,7 +164,9 @@ elif [[ $VARIANT == cu102* ]]; then
)
ml_files=( \
"libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}_${LIBCUDNN_VERSION}_amd64.deb" \
"libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
+ "libnccl2_${LIBNCCL_VERSION}_amd64.deb" \
)
elif [[ $VARIANT == cu101* ]]; then
cuda_files=( \
@@ -216,7 +191,9 @@ elif [[ $VARIANT == cu101* ]]; then
)
ml_files=( \
"libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}_${LIBCUDNN_VERSION}_amd64.deb" \
"libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
+ "libnccl2_${LIBNCCL_VERSION}_amd64.deb" \
)
elif [[ $VARIANT == cu100* ]]; then
cuda_files=( \
@@ -241,125 +218,9 @@ elif [[ $VARIANT == cu100* ]]; then
)
ml_files=( \
"libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libcudnn${LIBCUDNN_MAJOR}_${LIBCUDNN_VERSION}_amd64.deb" \
"libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
- )
-elif [[ $VARIANT == cu92* ]]; then
- cuda_files=( \
- "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvcc-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- )
- ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
- "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
- )
-elif [[ $VARIANT == cu91* ]]; then
- cuda_files=( \
- "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvcc-${CUDA_MAJOR_DASH}_9.1.85.2-1_amd64.deb" \
- "cuda-nvtx-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- )
- ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
- "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
- )
-elif [[ $VARIANT == cu90* ]]; then
- cuda_files=( \
- "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cublas-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \
- "cuda-cublas-dev-${CUDA_MAJOR_DASH}_9.0.176.4-1_amd64.deb" \
- "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- )
- ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
- "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
- )
-elif [[ $VARIANT == cu80* ]]; then
- cuda_files=( \
- "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- )
- ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
- "libnccl-dev_${LIBNCCL_VERSION}_amd64.deb" \
- )
-elif [[ $VARIANT == cu75* ]]; then
- cuda_files=( \
- "cuda-core-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cublas-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cublas-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-cudart-dev-${CUDA_MAJOR_DASH}_${CUDA_PATCH_VERSION}_amd64.deb" \
- "cuda-curand-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-curand-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cufft-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-nvrtc-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-cusolver-dev-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "cuda-misc-headers-${CUDA_MAJOR_DASH}_${CUDA_VERSION}_amd64.deb" \
- "libcuda1-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- "nvidia-${LIBCUDA_MAJOR}_${LIBCUDA_VERSION}_amd64.deb" \
- )
- ml_files=( \
- "libcudnn${LIBCUDNN_MAJOR}-dev_${LIBCUDNN_VERSION}_amd64.deb" \
+ "libnccl2_${LIBNCCL_VERSION}_amd64.deb" \
)
fi
@@ -377,7 +238,12 @@ if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION}
]]; then
for item in ${ml_files[*]}
do
echo "Installing $item"
- curl -sL
"http://developer.download.nvidia.com/compute/machine-learning/repos/${os_id}/x86_64/${item}";
-o package.deb
+ if [[ $item == libnccl* ]] && [[ $VARIANT == cu112* ]] ; then
+ echo "variant ${VARIANT} and installing ${item}"
+ curl -sL
"http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/${item}";
-o package.deb
+ else
+ curl -sL
"http://developer.download.nvidia.com/compute/machine-learning/repos/${os_id}/x86_64/${item}";
-o package.deb
+ fi
dpkg -X package.deb ${prefix}
rm package.deb
done
@@ -390,9 +256,7 @@ if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION}
]]; then
done
fi
cp -f ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h
${prefix}/include/cudnn.h
- ln -sf libcudnn_static_v${LIBCUDNN_MAJOR}.a
${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a
- cp -f ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a
${prefix}/lib/
+ ln -sf ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.so.${LIBCUDNN_MAJOR}
${prefix}/lib/libcudnn.so
cp -f ${prefix}/usr/include/nccl.h ${prefix}/include/nccl.h
- ln -sf libnccl_static.a ${prefix}/usr/lib/x86_64-linux-gnu/libnccl.a
fi
