SINGA-242 Compile all source files into a single library 1. Updated the cmake file for detecting glog automatically. 2. Add instructions in installation.md for PySINGA
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3d181110 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3d181110 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3d181110 Branch: refs/heads/master Commit: 3d181110ffa1b84708b3c4997c2d3c1c3b01a44a Parents: 2008446 Author: Wei Wang <[email protected]> Authored: Tue Aug 30 21:15:40 2016 +0800 Committer: Wei Wang <[email protected]> Committed: Tue Aug 30 22:29:01 2016 +0800 ---------------------------------------------------------------------- cmake/Dependencies.cmake | 24 +++++----- doc/en/docs/installation.md | 89 +++++++++++++++++++++++++++-------- examples/cifar10/CMakeLists.txt | 18 +++---- examples/cifar10/README.md | 9 +--- examples/imagenet/CMakeLists.txt | 14 +++--- src/core/device/cuda_gpu.cc | 2 +- 6 files changed, 100 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/cmake/Dependencies.cmake ---------------------------------------------------------------------- diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index b5fda6d..5fb7578 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -6,15 +6,15 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# SET(SINGA_LINKER_LIBS "") @@ -26,12 +26,12 @@ MESSAGE(STATUS "proto libs " ${PROTOBUF_LIBRARIES}) LIST(APPEND SINGA_LINKER_LIBS ${PROTOBUF_LIBRARIES}) INCLUDE("cmake/Protobuf.cmake") -#FIND_PACKAGE(Glog) -#IF(GLOG_FOUND) -# MESSAGE(STATUS "GLOG FOUND at ${GLOG_INCLUDE_DIR}") -# ADD_DEFINITIONS("-DUSE_GLOG") -# LIST(APPEND SINGA_LINKER_LIBS ${GLOG_LIBRARIES}) -#ENDIF() +FIND_PACKAGE(Glog) +IF(GLOG_FOUND) + MESSAGE(STATUS "GLOG FOUND at ${GLOG_INCLUDE_DIR}") + ADD_DEFINITIONS("-DUSE_GLOG") + LIST(APPEND SINGA_LINKER_LIBS ${GLOG_LIBRARIES}) +ENDIF() IF(USE_LMDB) FIND_PACKAGE(LMDB REQUIRED) @@ -69,9 +69,9 @@ IF(USE_OPENCL) ENDIF() ENDIF() -FIND_PACKAGE(Glog REQUIRED) -INCLUDE_DIRECTORIES(SYSTEM ${GLOG_INCLUDE_DIRS}) -LIST(APPEND SINGA_LINKER_LIBS ${GLOG_LIBRARIES}) +#FIND_PACKAGE(Glog REQUIRED) +#INCLUDE_DIRECTORIES(SYSTEM ${GLOG_INCLUDE_DIRS}) +#LIST(APPEND SINGA_LINKER_LIBS ${GLOG_LIBRARIES}) #MESSAGE(STATUS "Found glog at ${GLOG_INCLUDE_DIRS}") IF(USE_OPENCV) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/doc/en/docs/installation.md ---------------------------------------------------------------------- diff --git a/doc/en/docs/installation.md b/doc/en/docs/installation.md index 3fb6ba9..08508dc 100755 --- a/doc/en/docs/installation.md +++ b/doc/en/docs/installation.md @@ -9,36 +9,81 @@ ### Optional -* glog -* opencv (tested with 2.4.8) -* lmdb (tested with 0.9) * cuda (tested with 6.5, 7.0 and 7.5) * cudnn (v4 and v5) +* opencv (tested with 2.4.8) +* lmdb (tested with 0.9) +* glog -PySINGA has additional dependencies +PySINGA (the Python binding) has additional dependencies * python(==2.7) * pip(>=1.5) -* swig(>=3.0) +* swig(>=3.0.10) * numpy(>=1.11.0) -* openblas (>=0.2.10) Users are encouraged to install the cuda and [cudnn](https://developer.nvidia.com/cudnn) for running SINGA on GPUs to get better performance. -Most of the dependent libraries could be installed via package mangers like -apt-get or homebrew. +Most of the dependent libraries could be installed from source or via package mangers like +apt-get, homebrew, pip and anaconda. Please refer to FAQ for problems caused by the path setting of the dependent libraries. + + +### apt-get +The following instructions are tested on Ubuntu 14.04 for installing dependent libraries. + + # required libraries + $ sudo apt-get install libprotobuf-dev libopenblas-dev protobuf-compiler + + # optional libraries + $ sudo apt-get install python2.7-dev python-pip python-numpy + $ sudo apt-get install llibopencv-dev ibgoogle-glog-dev liblmdb-dev + +Please note that PySINGA requires swig >=3.0, which could be installed via +apt-get on Ubuntu 16.04; but it has to be installed from source for other Ubuntu versions including 14.04. + +### homebrew +The following instructions are tested on Mac OS X Yosemite (10.10.5) for installing dependent libraries. + + # required libraries + $ brew tap homebrew/science + $ brew install openblas + $ brew install protobuf260 - # for ubuntu users, tested on 14.04 - sudo apt-get install libprotobuf-dev libopenblas-dev libopencv-dev protobuf-compiler libgoogle-glog-dev liblmdb-dev python2.7-dev python-pip python-numpy + # optional libraries + $ brew tap homebrew/python + $ brew install python + $ brew install opencv + $ brew install -vd glog lmdb - # for Mac OS users - brew install -vd glog lmdb - brew tap homebrew/science - brew install opencv - brew install openblas - brew tap homebrew/python - brew install python - brew install numpy --with-openblas +By default, openblas is installed into /usr/local/opt/openblas. To let the compiler (and cmake) know the openblas +path, please export + + $ export CMAKE_INCLUDE_PATH=/usr/local/opt/openblas/include:$CMAKE_INCLUDE_PATH + $ export CMAKE_LIBRARY_PATH=/usr/local/opt/openblas/library:$CMAKE_LIBRARY_PATH + +To let the runtime know the openblas path, please export + + $ export LD_LIBRARY_PATH=/usr/local/opt/openblas/library:$LD_LIBRARY_PATH + +### pip and anaconda for PySINGA +pip and anaconda could be used to install python packages, e.g. numpy. +To use pip with virtual environment, + + # install virtualenv + $ pip install virtualenv + $ virtualenv pysinga + $ source pysinga/bin/activate + $ pip install numpy + +To use anaconda with virtual environment, + + $ conda create --name pysinga python=2 + $ source activate pysinga + $ conda install numpy + +After installing numpy, please export the header path of numpy.i as + + $ export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH ## Install PySINGA @@ -95,7 +140,7 @@ To build cnmem into the wheel file, please change CMakeLists.txt by replacing ### From the downloaded `tar.gz` file: Extract the downloaded. If using CUDA, CNMeM needs to be fetched: - + $ cd $SINGA_ROOT/lib/cnmem/ $ git clone https://github.com/NVIDIA/cnmem @@ -240,3 +285,9 @@ To be added. A: It could be caused by the `PYTHONPATH` which should be set to empty when you are using virtual environment to avoid the conflicts with the path of the virtual environment. + +* Q: When compiling PySINGA from source, there is a compilation error due to the missing of <numpy/objectarray.h> + + A: Please install numpy and export the path of numpy header files as + + $ export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/examples/cifar10/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/examples/cifar10/CMakeLists.txt b/examples/cifar10/CMakeLists.txt index 313c0eb..1f29f2f 100644 --- a/examples/cifar10/CMakeLists.txt +++ b/examples/cifar10/CMakeLists.txt @@ -6,31 +6,31 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) IF(USE_CUDNN) ADD_EXECUTABLE(alexnet alexnet.cc) -ADD_DEPENDENCIES(alexnet singa_core singa_model singa_utils) -TARGET_LINK_LIBRARIES(alexnet singa_core singa_utils singa_model protobuf ${SINGA_LIBKER_LIBS}) +ADD_DEPENDENCIES(alexnet singa) +TARGET_LINK_LIBRARIES(alexnet singa protobuf ${SINGA_LIBKER_LIBS}) ADD_EXECUTABLE(alexnet-parallel alexnet-parallel.cc) -ADD_DEPENDENCIES(alexnet-parallel singa_core singa_model singa_utils) -TARGET_LINK_LIBRARIES(alexnet-parallel singa_core singa_utils singa_model protobuf ${SINGA_LIBKER_LIBS}) +ADD_DEPENDENCIES(alexnet-parallel singa) +TARGET_LINK_LIBRARIES(alexnet-parallel singa protobuf ${SINGA_LIBKER_LIBS}) SET_TARGET_PROPERTIES(alexnet-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") ADD_EXECUTABLE(vgg-parallel vgg-parallel.cc) -ADD_DEPENDENCIES(vgg-parallel singa_core singa_model singa_utils) -TARGET_LINK_LIBRARIES(vgg-parallel singa_core singa_utils singa_model protobuf ${SINGA_LIBKER_LIBS}) +ADD_DEPENDENCIES(vgg-parallel singa) +TARGET_LINK_LIBRARIES(vgg-parallel singa protobuf ${SINGA_LIBKER_LIBS}) SET_TARGET_PROPERTIES(vgg-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") ENDIF(USE_CUDNN) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/examples/cifar10/README.md ---------------------------------------------------------------------- diff --git a/examples/cifar10/README.md b/examples/cifar10/README.md index bd5ed7d..de122f7 100644 --- a/examples/cifar10/README.md +++ b/examples/cifar10/README.md @@ -56,7 +56,7 @@ averaged on the host CPU device and then be applied to update the parameters. ./run-parallel.sh -4. vgg-parallel.cc. It train the VGG model using the CPP APIs on two CudaGPU devices similar to alexnet-parallel.cc. +4. vgg-parallel.cc. It trains the VGG model using the CPP APIs on two CudaGPU devices similar to alexnet-parallel.cc. ### Prediction @@ -68,10 +68,3 @@ The net is created by loading the previously trained model; Images consist of a numpy array of images (one row per image); dev is the training device, e.g., a CudaGPU device or the host CppCPU device; topk labels of each image would be returned. - - - - - - - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/examples/imagenet/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/examples/imagenet/CMakeLists.txt b/examples/imagenet/CMakeLists.txt index 465245a..fbb7235 100644 --- a/examples/imagenet/CMakeLists.txt +++ b/examples/imagenet/CMakeLists.txt @@ -6,15 +6,15 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) @@ -23,12 +23,12 @@ IF(USE_CUDNN) IF(USE_OPENCV) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp ") ADD_EXECUTABLE(imagenet alexnet.cc) - ADD_DEPENDENCIES(imagenet singa_core singa_model singa_utils singa_io) - TARGET_LINK_LIBRARIES(imagenet singa_core singa_utils singa_model singa_io protobuf ${SINGA_LIBKER_LIBS}) + ADD_DEPENDENCIES(imagenet singa) + TARGET_LINK_LIBRARIES(imagenet singa protobuf ${SINGA_LIBKER_LIBS}) ADD_EXECUTABLE(createdata ilsvrc12.cc) - ADD_DEPENDENCIES(createdata singa_core singa_io singa_model singa_utils) - TARGET_LINK_LIBRARIES(createdata singa_core singa_utils singa_io singa_model protobuf ${SINGA_LIBKER_LIBS}) + ADD_DEPENDENCIES(createdata singa) + TARGET_LINK_LIBRARIES(createdata singa protobuf ${SINGA_LIBKER_LIBS}) #SET_TARGET_PROPERTIES(createdata PROPERTIES LINK_FLAGS "${LINK_FLAGS}") ENDIF(USE_OPENCV) ENDIF(USE_CUDNN) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3d181110/src/core/device/cuda_gpu.cc ---------------------------------------------------------------------- diff --git a/src/core/device/cuda_gpu.cc b/src/core/device/cuda_gpu.cc index 0164752..f6603d3 100644 --- a/src/core/device/cuda_gpu.cc +++ b/src/core/device/cuda_gpu.cc @@ -53,7 +53,7 @@ CudaGPU::CudaGPU(int id) : Device(id, kNumCudaStream) { CudaGPU::CudaGPU(int id, std::shared_ptr<DeviceMemPool> pool) : Device(id, kNumCudaStream) { - CHECK_NE(pool, nullptr); + CHECK(pool != nullptr); pool_ = pool; Setup(); }
