SINGA-346 Update cudnn from V5 to V7 Export numpy path by cmake; no need to export it into a env var now.
replace PYTHON3 with USE_PYTHON3 one bug left due to the unicode and byte string problem from py2 to py3. which could be resolved by defining SWIG_PYTHON_2_UNICODE in *.i files. It converts byte string in python2 into unicode and then to c++ std::string. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/9090160e Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/9090160e Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/9090160e Branch: refs/heads/master Commit: 9090160ed28f2449d73e3bf82fa8cd8c93312eab Parents: 4ccb72e Author: Wang Wei <[email protected]> Authored: Wed Apr 18 14:53:15 2018 +0800 Committer: Wang Wei <[email protected]> Committed: Wed Apr 18 22:44:22 2018 +0800 ---------------------------------------------------------------------- CMakeLists.txt | 14 ++++---- cmake/Dependencies.cmake | 12 +++---- doc/en/docs/installation.md | 2 +- python/CMakeLists.txt | 10 ++++-- python/singa/layer.py | 4 +-- python/singa/net.py | 24 +++++++------ python/singa/optimizer.py | 16 ++++----- tool/conda/README.md | 2 +- tool/conda/build.sh | 16 +++++---- tool/docker/README.md | 33 ++++++------------ tool/docker/build.sh | 19 +++-------- tool/docker/devel/conda/cuda/Dockerfile | 2 +- tool/docker/devel/native/centos6/Dockerfile | 7 ++-- tool/docker/devel/native/ubuntu/Dockerfile | 36 -------------------- .../devel/native/ubuntu/cuda/py3/Dockerfile | 3 +- tool/docker/runtime/Dockerfile | 2 +- tool/jenkins/gen_doc.sh | 4 ++- tool/jenkins/test.sh | 13 +++++-- 18 files changed, 91 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cdde49..b630497 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,19 +52,21 @@ SET(SINGA_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include;${PROJECT_BINARY_DIR}") INCLUDE_DIRECTORIES(${SINGA_INCLUDE_DIR}) -#OPTION(USE_CBLAS "Use CBlas libs" ON) + OPTION(USE_CUDA "Use Cuda libs" OFF) +OPTION(ENABLE_TEST "Enable unit test" OFF) +OPTION(USE_PYTHON "Generate py wrappers" ON) +OPTION(USE_PYTHON3 "Python 3x" OFF) + OPTION(USE_CUDNN "Use Cudnn libs" ON) OPTION(USE_OPENCV "Use opencv" OFF) OPTION(USE_LMDB "Use LMDB libs" OFF) -OPTION(USE_PYTHON "Generate py wrappers" ON) OPTION(USE_JAVA "Generate java wrappers" OFF) OPTION(USE_OPENCL "Use OpenCL" OFF) OPTION(ENABLE_DIST "Enable distributed training" OFF) -OPTION(ENABLE_TEST "Enable unit test" OFF) OPTION(DISABLE_WARNINGS "Disable warnings under windows" ON) OPTION(USE_MODULES "Compile dependent libs as submodules together with singa" OFF) -#OPTION(USE_SHARED_LIBS "Use shared library" OFF) + # TODO: remove all USE_CBLAS in codes SET(USE_CBLAS ON) @@ -192,7 +194,7 @@ IF(PACKAGE) SET(CORE_DEPENDENCIES "libgoogle-glog-dev, libprotobuf-dev, libopenblas-dev, libstdc++6, libc6") ENDIF() - IF(PYTHON3) + IF(USE_PYTHON3) SET(PYTHON_DEPENDENCIES "${CORE_DEPENDENCIES}, python3, python3-dev, python3-pip, python3-numpy, python3-pillow, python3-matplotlib") ELSE() SET(PYTHON_DEPENDENCIES "${CORE_DEPENDENCIES}, python-dev, libpython2.7, python-pip, python-numpy, python-pillow") @@ -210,7 +212,7 @@ IF(PACKAGE) SET(CPACK_DEBIAN_PACKAGE_DEPENDS ${PYTHON_DEPENDENCIES}) SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_SOURCE_DIR}/tool/debian/postinst" ) SET(CPACK_DEBIAN_PACKAGE_PREDEPENDS "ca-certificates") - IF(PYTHON3) + IF(USE_PYTHON3) SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_SOURCE_DIR}/tool/debian/postinst" ) IF (USE_CUDA) SET(CPACK_DEBIAN_PACKAGE_NAME "python3-singa-cuda") http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/cmake/Dependencies.cmake ---------------------------------------------------------------------- diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 0b7d8a0..187d467 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -124,14 +124,14 @@ ENDIF() #MESSAGE(STATUS "link lib : " ${SINGA_LINKER_LIBS}) IF(USE_PYTHON) - IF(PYTHON3) - set(Python_ADDITIONAL_VERSIONS 3.6) - FIND_PACKAGE(PythonLibs 3 REQUIRED) + IF(USE_PYTHON3) + set(Python_ADDITIONAL_VERSIONS 3.6 3.5 3.4) FIND_PACKAGE(PythonInterp 3 REQUIRED) - FIND_PACKAGE(SWIG 3.0.10 REQUIRED) - ELSE() - FIND_PACKAGE(PythonLibs 2.7 REQUIRED) + FIND_PACKAGE(PythonLibs 3 REQUIRED) + FIND_PACKAGE(SWIG 3.0.8 REQUIRED) + ELSE() FIND_PACKAGE(PythonInterp 2.7 REQUIRED) + FIND_PACKAGE(PythonLibs 2.7 REQUIRED) FIND_PACKAGE(SWIG 3.0.8 REQUIRED) ENDIF() ENDIF() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/doc/en/docs/installation.md ---------------------------------------------------------------------- diff --git a/doc/en/docs/installation.md b/doc/en/docs/installation.md index 0d0a33e..dd5e961 100755 --- a/doc/en/docs/installation.md +++ b/doc/en/docs/installation.md @@ -109,7 +109,7 @@ The following libraries are optional * `USE_CUDA=ON`, used if CUDA and cuDNN is available * `USE_PYTHON=ON`, used for compiling PySINGA * `USE_OPENCL=ON`, used for compiling with OpenCL support - * `PYTHON3=ON`, used for compiling with Python 3 support. (The default is Python 2) + * `USE_PYTHON3=ON`, used for compiling with Python 3 support. (The default is Python 2) * `PACKAGE=ON`, used for building the Debian package 3. compile the code, e.g., `make` http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f5a1484..75e39f3 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -94,7 +94,7 @@ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa/proto) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/rafiki) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/api) -IF(PYTHON3) +IF(USE_PYTHON3) SET(SWIG_PYTHON3 "-py3") ELSE() SET(SWIG_PYTHON3 "") @@ -113,6 +113,12 @@ file(GLOB_RECURSE python_source_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.py) create_symlinks(${python_source_files}) +execute_process( + COMMAND ${PYTHON_EXECUTABLE} -c "from __future__ import print_function; import numpy; print(numpy.get_include())" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR) + +#message(status "numpy path ${NUMPY_INCLUDE_DIR}") + IF(USE_CUDA) # remain this custom command to avoid cuda objs can't find ADD_CUSTOM_COMMAND( @@ -123,7 +129,7 @@ ENDIF(USE_CUDA) ADD_LIBRARY(_singa_wrap SHARED $<TARGET_OBJECTS:singa_objects> ${python_srcs} ${proto_pys} ${global_cuda_objs}) TARGET_LINK_LIBRARIES(_singa_wrap ${SINGA_LINKER_LIBS} ${PYTHON_LIBRARIES}) -TARGET_INCLUDE_DIRECTORIES(_singa_wrap PRIVATE ${PYTHON_INCLUDE_DIRS}) +TARGET_INCLUDE_DIRECTORIES(_singa_wrap PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) SET_TARGET_PROPERTIES(_singa_wrap PROPERTIES PREFIX "" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/python/singa/layer.py ---------------------------------------------------------------------- diff --git a/python/singa/layer.py b/python/singa/layer.py index 516e778..c6ff30a 100644 --- a/python/singa/layer.py +++ b/python/singa/layer.py @@ -102,7 +102,7 @@ class Layer(object): self.param_specs = [] else: self.conf = conf - self.name = str(conf.name) + self.name = conf.name self.caffe_layer() self.param_specs = [] @@ -154,7 +154,7 @@ class Layer(object): if self.conf.type == 'InnerProduct' or self.conf.type == 14: self.layer = _create_layer(engine, 'Dense') else: - self.layer = _create_layer(engine, str(self.conf.type)) + self.layer = _create_layer(engine, self.conf.type) def get_output_sample_shape(self): '''Called after setup to get the shape of the output sample(s). http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/python/singa/net.py ---------------------------------------------------------------------- diff --git a/python/singa/net.py b/python/singa/net.py index 8f5ba97..fa223b2 100644 --- a/python/singa/net.py +++ b/python/singa/net.py @@ -18,6 +18,7 @@ Nerual net class for constructing the nets using layers and providing access functions for net info, e.g., parameters. + Example usages:: from singa import net as ffnet @@ -175,11 +176,11 @@ class FeedForwardNet(object): return [spec.name for spec in self.param_specs()] def train(self, x, y): - '''Run BP for one iteration. - This method is deprecated. It is only kept for backward compatibility. - The name of this method is confusing since it does not update parameters. - Please use backprob() instead. - The back progagation algorithm computes gradients but it does not train. + '''Run BP for one iteration. + This method is deprecated. It is only kept for backward compatibility. + The name of this method is confusing since it does not update parameters. + Please use backprob() instead. + The back progagation algorithm computes gradients but it does not train. ''' return backprob(x, y) @@ -295,7 +296,8 @@ class FeedForwardNet(object): dictionary: layer name -> output tensor(s) ''' if self.ordered_layers is None: - self.ordered_layers = self.topo_sort(self.layers, self.src_of_layer) + self.ordered_layers = self.topo_sort( + self.layers, self.src_of_layer) if type(x) is dict: input_of_layer = x else: @@ -326,7 +328,7 @@ class FeedForwardNet(object): outs = output_of_layer[src.name] if type(outs) == list: assert len(outs) > 0, \ - 'the output from layer %s is empty' % src.name + 'the output from layer %s is empty' % src.name inputs.append(outs[0]) outs.pop(0) if len(outs) == 0: @@ -406,7 +408,7 @@ class FeedForwardNet(object): outputs = output_of_layer[dst.name] if type(outputs) == list: assert len(outputs) > 0, \ - 'the gradient from layer %s is empty' % dst.name + 'the gradient from layer %s is empty' % dst.name inputs.append(outputs[0]) outputs.pop(0) else: @@ -418,7 +420,7 @@ class FeedForwardNet(object): outs, pgrads = cur.backward(kTrain, inputs) if verbose: disp_src = '+'.join( - [dst.name for dst in self.dst_of_layer[cur.name]]) + [dst.name for dst in self.dst_of_layer[cur.name]]) disp_src += '-->' + cur.name if type(outs) is list: print('%s: %s' % (disp_src, @@ -477,7 +479,7 @@ class FeedForwardNet(object): if version < 1101: idx = name.rfind('/') assert idx > 0, '/ must be in the parameter name' - name = name[:idx] + '_' + name[idx+1:] + name = name[:idx] + '_' + name[idx + 1:] return name if use_pickle: @@ -515,5 +517,5 @@ class FeedForwardNet(object): except AssertionError as err: print('Error from copying values for param: %s' % name) print(('shape of param vs checkpoint', - val.shape, params[name].shape)) + val.shape, params[name].shape)) raise err http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/python/singa/optimizer.py ---------------------------------------------------------------------- diff --git a/python/singa/optimizer.py b/python/singa/optimizer.py index 975641a..5cb02e5 100644 --- a/python/singa/optimizer.py +++ b/python/singa/optimizer.py @@ -206,7 +206,7 @@ class SGD(Optimizer): if self.momentum is not None: conf.momentum = self.momentum conf.type = 'sgd' - self.opt = singa.CreateOptimizer('SGD'.encode()) + self.opt = singa.CreateOptimizer('SGD') self.opt.Setup(conf.SerializeToString()) def apply_with_lr(self, epoch, lr, grad, value, name, step=-1): @@ -216,7 +216,7 @@ class SGD(Optimizer): epoch, value, grad, name, step) if name is not None and name in self.learning_rate_multiplier: lr = lr * self.learning_rate_multiplier[name] - self.opt.Apply(epoch, lr, name.encode(), grad.data, + self.opt.Apply(epoch, lr, name, grad.data, value.data) return value @@ -235,7 +235,7 @@ class Nesterov(Optimizer): if self.momentum is not None: conf.momentum = momentum conf.type = 'nesterov' - self.opt = singa.CreateOptimizer('Nesterov'.encode()) + self.opt = singa.CreateOptimizer('Nesterov') self.opt.Setup(conf.SerializeToString()) def apply_with_lr(self, epoch, lr, grad, value, name, step=-1): @@ -246,7 +246,7 @@ class Nesterov(Optimizer): epoch, value, grad, name, step) if name is not None and name in self.learning_rate_multiplier: lr = lr * self.learning_rate_multiplier[name] - self.opt.Apply(epoch, lr, name.encode(), grad.data, + self.opt.Apply(epoch, lr, name, grad.data, value.data) return value @@ -268,7 +268,7 @@ class RMSProp(Optimizer): conf = model_pb2.OptimizerConf() conf.rho = rho conf.delta = epsilon - self.opt = singa.CreateOptimizer('RMSProp'.encode()) + self.opt = singa.CreateOptimizer('RMSProp') self.opt.Setup(conf.SerializeToString()) def apply_with_lr(self, epoch, lr, grad, value, name, step=-1): @@ -279,7 +279,7 @@ class RMSProp(Optimizer): epoch, value, grad, name, step) if name is not None and name in self.learning_rate_multiplier: lr = lr * self.learning_rate_multiplier[name] - self.opt.Apply(step, lr, name.encode(), grad.data, + self.opt.Apply(step, lr, name, grad.data, value.data) return value @@ -300,7 +300,7 @@ class AdaGrad(Optimizer): conf = model_pb2.OptimizerConf() conf.delta = epsilon conf.type = 'adagrad' - self.opt = singa.CreateOptimizer('AdaGrad'.encode()) + self.opt = singa.CreateOptimizer('AdaGrad') self.opt.Setup(conf.SerializeToString()) def apply_with_lr(self, epoch, lr, grad, value, name, step=-1): @@ -311,7 +311,7 @@ class AdaGrad(Optimizer): epoch, value, grad, name, step) if name is not None and name in self.learning_rate_multiplier: lr = lr * self.learning_rate_multiplier[name] - self.opt.Apply(epoch, lr, name.encode(), grad.data, + self.opt.Apply(epoch, lr, name, grad.data, value.data) return value http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/conda/README.md ---------------------------------------------------------------------- diff --git a/tool/conda/README.md b/tool/conda/README.md index f162cf9..19b55b3 100644 --- a/tool/conda/README.md +++ b/tool/conda/README.md @@ -24,7 +24,7 @@ this folder should include a subfolder `include/cudnn.h` for the header file, an After exporting the environment variables, execute the following command to compile Singa and package it - conda-build . + conda-build . --python 3.6 (or 2.7) You will see the package path from the screen output. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/conda/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/build.sh b/tool/conda/build.sh index cb4592c..baa7ac3 100644 --- a/tool/conda/build.sh +++ b/tool/conda/build.sh @@ -16,15 +16,14 @@ # # to compile swig api files which depdend on numpy.i -export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH +# export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH # to let cmake use the dependent libs installed by conda, including python export CMAKE_PREFIX_PATH=$PREFIX:$CMAKE_PREFIX_PATH export CMAKE_INCLUDE_PATH=$PREFIX/include:$CMAKE_INCLUDE_PATH export CMAKE_LIBRARY_PATH=$PREFIX/lib:$CMAKE_LIBRARY_PATH -mkdir build -cd build + USE_CUDA=OFF if [ -z ${CUDNN_PATH+x} ]; then USE_CUDA=ON @@ -32,10 +31,13 @@ if [ -z ${CUDNN_PATH+x} ]; then cp -P $CUDNN_PATH/lib64/libcudnn.so* $PREFIX/lib/ fi -PYTHON3=OFF +USE_PYTHON3=OFF # PY3K is set by conda -if [ "$PY3K" == "1" ]; then PYTHON3=ON; fi -echo "PY3K = $PY3K" -cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DPYTHON3=$PYTHON3 .. +if [ "$PY3K" == "1" ]; then USE_PYTHON3=ON; fi + + +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DUSE_PYTHON3=$USE_PYTHON3 .. make make install http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/README.md ---------------------------------------------------------------------- diff --git a/tool/docker/README.md b/tool/docker/README.md index d61554f..06ce0e9 100644 --- a/tool/docker/README.md +++ b/tool/docker/README.md @@ -2,36 +2,23 @@ ## Availabe images -TO BE UPDATED. -| Tag | OS version | devel/runtime | Device|CUDA/CUDNN| -|:----|:-----------|:--------------|:------|:---------| -|runtime| Ubuntu16.04|runtime|CPU|-| -|runtime| Ubuntu16.04|runtime|CPU|-| -|runtime-cuda| Ubuntu16.04|runtime|GPU|CUDA8.0+CUDNN5| -|devel| Ubuntu16.04|devel|CPU|-| -|devel-cuda| Ubuntu16.04|devel|GPU|CUDA8.0+CUDNN5| +| Tag | OS version | devel/runtime | Device|CUDA/CUDNN|Python| +|:----|:-----------|:--------------|:------|:---------|:-----| +|runtime| Ubuntu16.04|runtime|CPU|-|3.6| +|conda-cuda9.0| Ubuntu16.04|devel|GPU|CUDA9.0+CUDNN7.1.2|3.6| +|cuda9.0-py2| Ubuntu16.04|devel|GPU|CUDA9.0+CUDNN7.1.2|2.7| +|cuda9.0-py3| Ubuntu16.04|devel|GPU|CUDA9.0+CUDNN7.1.2|3.6| + +runtime and conda-xxx image has installed miniconda3; +cudaxxx images have installed all depedent libs using apt-get. ## Usage docker pull nusdbsystem/singa:<Tag> docker run -it nusdbsystem/singa:<Tag> /bin/bash + nvidia-docker run -it nusdbsystem/singa:<Tag> /bin/bash -* For the *devel* images, the container has a `incubator-singa` folder in the root directory, -which has the latest SINGA code. The code has been compiled into `incubator-singa/build` directory and PySINGA has been installed. -* For the *runtime* images, the container has only installed the PySINGA. - -## Tag naming style - singa:devel|runtime[-OS][-CUDA|OPENCL][-CUDNN] -* devel: development images with all dependent libs' header files installed and SINGA's source code; -* runtime: the minimal images which can run SINGA programs. -* OS: ubuntu, ubuntu14.04, centos, centos6 -* CUDA: cuda, cuda8.0, cuda7.0 -* CUDNN: cudnn, cudnn5, cudnn4 -* OPENCL: opencl, opencl1.2 -By default, if the version is not included in the tag, the latest stable version is used. -The default OS is ubuntu. The version is the latest stable version (e.g., 16.04 for now). -For -cuda version, the **cudnn** is included by default. Their versions are also the latest stable version, i.e., cuda-8.0 and cudnn-5 for now. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/build.sh ---------------------------------------------------------------------- diff --git a/tool/docker/build.sh b/tool/docker/build.sh index bc4ec40..666c91f 100755 --- a/tool/docker/build.sh +++ b/tool/docker/build.sh @@ -23,24 +23,15 @@ # (used by Jenkins to avoid dangling images from multiple building) echo "###################" -echo "build singa:runtime" +echo "build singa:conda-cudax.y" echo "###################" -docker build tool/docker/runtime/ --force-rm -t nusdbsystem/singa:runtime -t nusdbsystem/singa:latest +# docker build tool/docker/devel/conda/cuda/ --force-rm -t nusdbsystem/singa:conda-cuda9.0-cudnn7.1.2 echo "###################" -echo "build singa:runtime-cuda" +echo "build singa:cudax.y" echo "###################" -docker build tool/docker/runtime/cuda --force-rm -t nusdbsystem/singa:runtime-cuda - -echo "###################" -echo "build singa:devel" -echo "###################" -docker build tool/docker/devel/ --force-rm -t nusdbsystem/singa:devel - -echo "###################" -echo "build singa:devel-cuda" -echo "###################" -docker build tool/docker/devel/cuda --force-rm -t nusdbsystem/singa:devel-cuda +docker build tool/docker/devel/native/ubuntu/cuda/py2 --force-rm -t nusdbsystem/singa:cuda9.0-cudnn7.1.2-py2 +docker build tool/docker/devel/native/ubuntu/cuda/py3 --force-rm -t nusdbsystem/singa:cuda9.0-cudnn7.1.2-py3 if [ $1 = "PUSH" ]; then echo "##########################################" http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/devel/conda/cuda/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/conda/cuda/Dockerfile b/tool/docker/devel/conda/cuda/Dockerfile index 4b62271..04923a7 100644 --- a/tool/docker/devel/conda/cuda/Dockerfile +++ b/tool/docker/devel/conda/cuda/Dockerfile @@ -16,7 +16,7 @@ # # Change tags to build with different cuda/cudnn versions: -FROM nvidia/cuda:9.0-devel-ubuntu16.04 +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 # install dependencies RUN apt-get update \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/devel/native/centos6/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/centos6/Dockerfile b/tool/docker/devel/native/centos6/Dockerfile index 0538c13..c9193a3 100644 --- a/tool/docker/devel/native/centos6/Dockerfile +++ b/tool/docker/devel/native/centos6/Dockerfile @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO(wangwei) install other libs and test. It has bugs now. + # Change tags to build with different cuda/cudnn versions: FROM nvidia/cuda:7.5-cudnn5-devel-centos6 @@ -22,11 +24,6 @@ RUN yum -y update && yum -y install git wget openssh-server cmake -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - # download singa source # RUN git clone https://github.com/apache/incubator-singa.git http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/devel/native/ubuntu/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/ubuntu/Dockerfile b/tool/docker/devel/native/ubuntu/Dockerfile deleted file mode 100644 index c19738d..0000000 --- a/tool/docker/devel/native/ubuntu/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Base unbuntu 16.04 image -FROM ubuntu:latest - -MAINTAINER incubator-singa [email protected] - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential git cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget\ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip --no-cache-dir install -U pip numpy setuptools - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} - -# download singa source -RUN git clone https://github.com/apache/incubator-singa.git - -# compile singa and install pysinga -RUN cd incubator-singa && mkdir build && cd build && cmake .. && make && cd python && pip install . && cd ../../ && rm -rf build http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile b/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile index 2ecd3a9..c1d36c4 100644 --- a/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile +++ b/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev protobuf-compiler wget openssh-server swig python3-dev python3-pip python3-setuptools\ && apt-get clean && apt-get autoremove && apt-get autoclean \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install -U pip3 wheel numpy setuptools unittest-xml-reporting protobuf future + && pip3 install -U wheel numpy setuptools unittest-xml-reporting protobuf future # install swig > 3.0.10 for ubuntu < 16.04 @@ -32,6 +32,7 @@ RUN apt-get update \ # ./configure && make && make install # set environment + # ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} # ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/docker/runtime/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/runtime/Dockerfile b/tool/docker/runtime/Dockerfile index ff7d380..679e824 100644 --- a/tool/docker/runtime/Dockerfile +++ b/tool/docker/runtime/Dockerfile @@ -36,7 +36,7 @@ RUN conda install -c conda-forge sphinx RUN conda install -c conda-forge sphinx_rtd_theme RUN conda install -c clinicalgraphics recommonmark=0.2.0 # TODO(wangwei) add the cpu build string -RUN conda install -c nusdbsystem singa +RUN conda install -c nusdbsystem singa-cpu RUN mkdir /var/run/sshd RUN echo 'root:singa' | chpasswd http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/jenkins/gen_doc.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/gen_doc.sh b/tool/jenkins/gen_doc.sh index cb78f0d..a9f2e1d 100644 --- a/tool/jenkins/gen_doc.sh +++ b/tool/jenkins/gen_doc.sh @@ -18,7 +18,9 @@ # * limitations under the License. # */ -# This script is used by Jenkins to update SINGA website + +# This script is used by Jenkins to update Singa website. +# Run this script in runtime docker container. echo Install PySinga, generate HTML files and update Singa website http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9090160e/tool/jenkins/test.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/test.sh b/tool/jenkins/test.sh index 039c15a..9b223f2 100644 --- a/tool/jenkins/test.sh +++ b/tool/jenkins/test.sh @@ -25,6 +25,7 @@ echo workspace: `pwd` echo OS version: `cat /etc/issue` echo kernal version: `uname -a` echo parameters: $1 +echo parameters: $2 COMMIT=`git rev-parse --short HEAD` echo COMMIT HASH: $COMMIT @@ -44,11 +45,19 @@ rm -rf build mkdir build # compile c++ code cd build -cmake -DUSE_CUDA=$CUDA -DENABLE_TEST=ON $EXTRA_ARGS ../ +if [ $2 = "PYTHON3" ]; then + cmake -DUSE_CUDA=$CUDA -DENABLE_TEST=ON -DUSE_PYTHON3=ON $EXTRA_ARGS ../ +else + cmake -DUSE_CUDA=$CUDA -DENABLE_TEST=ON $EXTRA_ARGS ../ +fi make # unit test cpp code ./bin/test_singa --gtest_output=xml:./gtest.xml # unit test python code cd ../test/python -PYTHONPATH=../../build/python/ python run.py +if [ $2 = "PYTHON3" ]; then + PYTHONPATH=../../build/python/ python3 run.py +else + PYTHONPATH=../../build/python/ python run.py +fi echo Job finished...
