cjolivier01 closed pull request #9672: CMake CUDA fixes + NCCL
URL: https://github.com/apache/incubator-mxnet/pull/9672
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cddbf725c2..2edf3c23c9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,35 +1,17 @@
cmake_minimum_required(VERSION 3.0.2)
-message(STATUS "CMake version '${CMAKE_VERSION}' using generator
'${CMAKE_GENERATOR}'")
-if(((${CMAKE_GENERATOR} MATCHES "Visual Studio.*") OR (${CMAKE_GENERATOR}
MATCHES "Xcode.*"))
- AND ((${CMAKE_VERSION} VERSION_GREATER "3.9.0") OR (${CMAKE_VERSION}
VERSION_EQUAL "3.9.0")))
- # Toolsets are only supported for Visual Studio and Xcode
- set(FIRST_CUDA TRUE)
-else()
- set(FIRST_CUDA FALSE)
-endif()
-include(cmake/Utils.cmake)
-#Some things have order. This must be put in front alone
-mxnet_option(USE_CUDA "Build with CUDA support" ON)
-mxnet_option(USE_OLDCMAKECUDA "Build with old cmake cuda" OFF)
-if(USE_CUDA)
- add_definitions(-DMSHADOW_USE_CUDA=1)
- IF(FIRST_CUDA AND (NOT USE_OLDCMAKECUDA))
- set(__cuda_toolset "7.5" "8.0" "9.0")
- set(CUDA_TOOLSET "8.0" CACHE STRING "Select CUDA Version.")
- set_property( CACHE CUDA_TOOLSET PROPERTY STRINGS "" ${__cuda_toolset} )
- set(CMAKE_GENERATOR_TOOLSET "cuda=${CUDA_TOOLSET},host=x64")
- project(mxnet C CXX CUDA)
- else()
- project(mxnet C CXX)
- set(FIRST_CUDA FALSE)
- endif()
-else()
- project(mxnet C CXX)
- add_definitions(-DMSHADOW_USE_CUDA=0)
+project(mxnet C CXX)
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
+ include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
endif()
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Utils.cmake)
+#Some things have order. This must be put in front alone
+mxnet_option(USE_CUDA "Build with CUDA support" ON)
+mxnet_option(USE_OLDCMAKECUDA "Build with old cmake cuda" OFF)
+mxnet_option(USE_NCCL "Use NVidia NCCL with CUDA" OFF)
mxnet_option(USE_OPENCV "Build with OpenCV support" ON)
mxnet_option(USE_OPENMP "Build with Openmp support" ON)
mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could
set CUDNN_ROOT for search path
@@ -37,7 +19,7 @@ mxnet_option(USE_LAPACK "Build with lapack support"
ON IF NOT MSVC)
mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
mxnet_option(USE_MKLML_MKL "Use MKLML variant of MKL (if MKL found)" ON
IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and
found)" OFF)
-mxnet_option(USE_OPERATOR_TUNING "Enable auto-tuning of operators" ON AND NOT
MSVC)
+mxnet_option(USE_OPERATOR_TUNING "Enable auto-tuning of operators" ON IF NOT
MSVC)
mxnet_option(USE_GPERFTOOLS "Build with GPerfTools support (if found)"
ON)
mxnet_option(USE_JEMALLOC "Build with Jemalloc support" ON)
mxnet_option(USE_PROFILER "Build with Profiler support" ON)
@@ -47,30 +29,49 @@ mxnet_option(USE_PLUGIN_CAFFE "Use Caffe Plugin" OFF)
mxnet_option(USE_CPP_PACKAGE "Build C++ Package" OFF)
mxnet_option(USE_MXNET_LIB_NAMING "Use MXNet library naming conventions." ON)
mxnet_option(USE_GPROF "Compile with gprof (profiling) flag" OFF)
+mxnet_option(USE_CXX14_IF_AVAILABLE "Build with C++14 if the compiler supports
it" OFF)
mxnet_option(USE_VTUNE "Enable use of Intel Amplifier XE (VTune)"
OFF) # one could set VTUNE_ROOT for search path
mxnet_option(ENABLE_CUDA_RTC "Build with CUDA runtime compilation
support" ON)
mxnet_option(INSTALL_EXAMPLES "Install the example source files." OFF)
mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF)
-
-
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
- include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
+if(USE_CUDA AND NOT USE_OLDCMAKECUDA)
+ message(STATUS "CMake version '${CMAKE_VERSION}' using generator
'${CMAKE_GENERATOR}'")
+ if(
+ (
+ (${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
+ OR (${CMAKE_GENERATOR} MATCHES "Xcode.*")
+ OR (${CMAKE_GENERATOR} STREQUAL "Unix Makefiles")
+ ) AND (
+ (${CMAKE_VERSION} VERSION_GREATER "3.9.0") OR (${CMAKE_VERSION}
VERSION_EQUAL "3.9.0")
+ )
+ )
+ set(FIRST_CUDA TRUE)
+ project(mxnet C CXX CUDA)
+ message(ERROR " foo")
+ else()
+ set(FIRST_CUDA FALSE)
+ set(USE_OLDCMAKECUDA TRUE)
+ project(mxnet C CXX)
+ endif()
+else()
+ project(mxnet C CXX)
endif()
+
if(MSVC)
set(SYSTEM_ARCHITECTURE x86_64)
else()
- EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE
SYSTEM_ARCHITECTURE)
+ execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE
SYSTEM_ARCHITECTURE)
endif()
-set(CMAKE_MODULE_PATH
"${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
+set(CMAKE_MODULE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
if("$ENV{VERBOSE}" STREQUAL "1")
message(STATUS " Verbose Makefile ACTIVATED")
- set(CMAKE_VERBOISE_MAKEFILE ON)
+ set(CMAKE_VERBOSE_MAKEFILE ON)
endif()
@@ -83,10 +84,14 @@ if(MSVC)
add_definitions(-DMXNET_EXPORTS)
add_definitions(-DNNVM_EXPORTS)
add_definitions(-DDMLC_STRICT_CXX11)
+ add_definitions(-DNOMINMAX)
set(CMAKE_C_FLAGS "/MP")
set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /bigobj")
else(MSVC)
include(CheckCXXCompilerFlag)
+ if(USE_CXX14_IF_AVAILABLE)
+ check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
+ endif()
check_cxx_compiler_flag("-std=c++11" SUPPORT_CXX11)
check_cxx_compiler_flag("-std=c++0x" SUPPORT_CXX0X)
check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
@@ -107,7 +112,11 @@ else(MSVC)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2")
endif()
set(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS})
- if(SUPPORT_CXX11)
+ if(SUPPORT_CXX14)
+ add_definitions(-DDMLC_USE_CXX11=1)
+ add_definitions(-DDMLC_USE_CXX14=1)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+ elseif(SUPPORT_CXX11)
add_definitions(-DDMLC_USE_CXX11=1)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
elseif(SUPPORT_CXX0X)
@@ -120,8 +129,8 @@ set(mxnet_LINKER_LIBS "")
if(USE_GPROF)
message(STATUS "Using GPROF")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -pg")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -pg")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -g -pg")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -g -pg")
set(CMAKE_LINK_LIBRARY_FILE_FLAG "${CMAKE_LINK_LIBRARY_FILE_FLAG} -g -pg")
endif()
@@ -170,12 +179,43 @@ endif()
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
-if(FIRST_CUDA)
+if(USE_CUDA)
+ find_package(CUDA REQUIRED)
+ add_definitions(-DMSHADOW_USE_CUDA=1)
+ if(FIRST_CUDA AND (NOT USE_OLDCMAKECUDA))
+ if(CUDA_TOOLSET STREQUAL "")
+ set(CUDA_TOOLSET "${CUDA_VERSION_STRING}")
+ endif()
+ set(CMAKE_GENERATOR_TOOLSET "cuda=${CUDA_TOOLSET},host=x64")
+ else()
+ set(FIRST_CUDA FALSE)
+ endif()
+ if(USE_NCCL)
+ find_package(NCCL)
+ if(NCCL_FOUND)
+ include_directories(${NCCL_INCLUDE_DIRS})
+ list(APPEND mxnet_LINKER_LIBS ${NCCL_LIBRARIES})
+ else()
+ message(WARNING "Could not find NCCL libraries")
+ endif()
+ endif()
+else()
+ add_definitions(-DMSHADOW_USE_CUDA=0)
+endif()
+
+if(NCCL_FOUND)
+ add_definitions(-DMXNET_USE_NCCL=1)
+else()
+ add_definitions(-DMXNET_USE_NCCL=0)
+endif()
+
+if(USE_CUDA AND FIRST_CUDA)
include(cmake/ChooseBlas.cmake)
include(mshadow/cmake/Utils.cmake)
include(cmake/FirstClassLangCuda.cmake)
+ include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
else()
- if(EXISTS ${PROJECT_SOURCE_DIR}/mshadow/cmake)
+ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
include(mshadow/cmake/mshadow.cmake)
include(mshadow/cmake/Utils.cmake)
include(mshadow/cmake/Cuda.cmake)
@@ -238,8 +278,8 @@ endif()
# ---[ jemalloc
if(USE_JEMALLOC)
- if(USE_GPERFTOOLS)
- message(ERROR "Only one of USE_JEMALLOC and USE_GPERFTOOLS can be defined
at once")
+ if(GPERFTOOLS_FOUND)
+ message(ERROR " Only one of USE_JEMALLOC and USE_GPERFTOOLS can be defined
at once")
endif()
find_package(JeMalloc)
if(JEMALLOC_FOUND)
@@ -295,12 +335,12 @@ elseif(UNIX)
endif()
# ---[ LAPack
-if(USE_LAPACK)
+if(USE_LAPACK AND NOT MSVC)
add_definitions(-DMXNET_USE_LAPACK=1)
list(APPEND mxnet_LINKER_LIBS lapack)
else(USE_LAPACK)
# Workaround for Windows until using new Jenkinsfile.
- if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
+ if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open" OR USE_BLAS STREQUAL "Open"
OR USE_BLAS STREQUAL "open")
add_definitions(-DMXNET_USE_LAPACK=1)
endif()
endif()
@@ -367,7 +407,8 @@ list(APPEND CUDA ${MSHADOW_CUDASOURCE})
# add source group
FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "nnvm/*.cc" "plugin/*.cc")
FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "mshadow/mshadow/*.h"
"plugin/*.h")
-FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh"
"plugin/*.cu" "plugin/*.cuh")
+FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh"
"plugin/*.cu"
+ "plugin/*.cuh" "3rdparty/cub/cub/*.cuh")
assign_source_group("Source" ${GROUP_SOURCE})
assign_source_group("Include" ${GROUP_Include})
assign_source_group("CUDA" ${GROUP_CUDA})
@@ -376,8 +417,6 @@ if(USE_PLUGINS_WARPCTC)
set(WARPCTC_INCLUDE "" CACHE PATH "WARPCTC include")
set(WARPCTC_LIB_DEBUG "" CACHE FILEPATH "WARPCTC lib")
set(WARPCTC_LIB_RELEASE "" CACHE FILEPATH "WARPCTC lib")
-
-
include_directories(SYSTEM ${WARPCTC_INCLUDE})
list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc"
"plugin/warpctc/*.h")
@@ -399,7 +438,7 @@ if(USE_PLUGIN_CAFFE)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/caffe)
# Need newer FindCUDA.cmake that correctly handles -std=c++11
cmake_minimum_required(VERSION 3.3)
- set(CAFFE_PATH ${PROJECT_SOURCE_DIR}/caffe)
+ set(CAFFE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/caffe)
else()
set(CAFFE_PATH $ENV{CAFFE_PATH})
endif()
@@ -456,6 +495,7 @@ if(USE_CUDA)
endif()
list(APPEND SOURCE ${CUDA})
add_definitions(-DMXNET_USE_CUDA=1)
+ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
else()
list(APPEND CUDA_INCLUDE_DIRS ${INCLUDE_DIRECTORIES})
# define preprocessor macro so that we will not include the generated
forcelink header
@@ -472,6 +512,8 @@ if(USE_CUDA)
list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") #
For fft operator
FIND_LIBRARY(CUDA_cusolver_LIBRARY nvrtc
"${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
list(APPEND mxnet_LINKER_LIBS
"${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver
+ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib/win32)
+ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib/x64)
else(MSVC)
list(APPEND mxnet_LINKER_LIBS cufft cusolver)
if(ENABLE_CUDA_RTC)
@@ -605,13 +647,13 @@ if(AUTO_INSTALL_DIR)
# ---[ Install Includes
add_custom_command(TARGET mxnet POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
- ${PROJECT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include
)
# ---[ Install Examples
add_custom_command(TARGET mxnet POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
- ${PROJECT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example
+ ${CMAKE_CURRENT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example
)
endif()
diff --git a/cmake/Modules/FindNCCL.cmake b/cmake/Modules/FindNCCL.cmake
new file mode 100644
index 0000000000..d0ae44e3fe
--- /dev/null
+++ b/cmake/Modules/FindNCCL.cmake
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Find the nccl libraries
+#
+# The following variables are optionally searched for defaults
+# NCCL_ROOT_DIR: Base directory where all NCCL components are found
+# NCCL_INCLUDE_DIR: Directory where NCCL header is found
+# NCCL_LIB_DIR: Directory where NCCL library is found
+#
+# The following are set after configuration is done:
+# NCCL_FOUND
+# NCCL_INCLUDE_DIRS
+# NCCL_LIBRARIES
+#
+# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks
+# install NCCL in the same location as the CUDA toolkit.
+# See https://github.com/caffe2/caffe2/issues/1601
+
+set(NCCL_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA NCCL")
+
+find_path(NCCL_INCLUDE_DIRS
+ NAMES nccl.h
+ HINTS
+ ${NCCL_INCLUDE_DIR}
+ ${NCCL_ROOT_DIR}
+ ${NCCL_ROOT_DIR}/include
+ ${CUDA_TOOLKIT_ROOT_DIR}/include
+ $ENV{NCCL_DIR}/include
+ )
+
+find_library(NCCL_LIBRARIES
+ NAMES nccl
+ HINTS
+ ${NCCL_LIB_DIR}
+ ${NCCL_ROOT_DIR}
+ ${NCCL_ROOT_DIR}/lib
+ ${NCCL_ROOT_DIR}/lib/x86_64-linux-gnu
+ ${NCCL_ROOT_DIR}/lib64
+ ${CUDA_TOOLKIT_ROOT_DIR}/lib64
+ $ENV{NCCL_DIR}/lib
+ )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS
NCCL_LIBRARIES)
+
+if(NCCL_FOUND)
+ message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library:
${NCCL_LIBRARIES})")
+ mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+endif()
+
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 8a85c094ad..4dd409d979 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -26,6 +26,7 @@ if(GTEST_FOUND AND NOT MSVC)
file(GLOB_RECURSE UNIT_TEST_SOURCE "cpp/*.cc" "cpp/*.h")
+ include_directories(${GTEST_INCLUDE_DIR})
include_directories(cpp/include)
if (NOT PRIVATE_RUNTIME_DIR)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
