SINGA-346 Update cudnn from V5 to V7 Add Dockerfiles for building Singa with py2 and py3 in a native way instead of using conda build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/4ccb72ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/4ccb72ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/4ccb72ee Branch: refs/heads/master Commit: 4ccb72ee803520259f7fdaa25e64ce2aa8eca0f2 Parents: 98e9d2f Author: Wang Wei <[email protected]> Authored: Tue Apr 17 23:36:01 2018 +0800 Committer: Wang Wei <[email protected]> Committed: Wed Apr 18 22:43:25 2018 +0800 ---------------------------------------------------------------------- src/model/layer/cudnn_convolution.cc | 2 +- tool/docker/devel/conda/cuda/Dockerfile | 12 +--- tool/docker/devel/native/ubuntu/cuda/Dockerfile | 61 -------------------- .../devel/native/ubuntu/cuda/py2/Dockerfile | 53 +++++++++++++++++ .../devel/native/ubuntu/cuda/py3/Dockerfile | 53 +++++++++++++++++ tool/jenkins/test.sh | 23 +++----- 6 files changed, 117 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/src/model/layer/cudnn_convolution.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/cudnn_convolution.cc b/src/model/layer/cudnn_convolution.cc index 40c7d33..8846746 100644 --- a/src/model/layer/cudnn_convolution.cc +++ b/src/model/layer/cudnn_convolution.cc @@ -107,7 +107,7 @@ void CudnnConvolution::InitCudnn(const Tensor &input) { CUDNN_CHECK(cudnnGetConvolutionBackwardFilterAlgorithm( ctx->cudnn_handle, x_desc_, y_desc_, conv_desc_, filter_desc_, bwd_filt_pref, workspace_byte_limit_, &bp_filter_alg_)); - # deprecated in cudnn v7 + // deprecated in cudnn v7 CUDNN_CHECK(cudnnGetConvolutionBackwardDataAlgorithm( ctx->cudnn_handle, filter_desc_, y_desc_, conv_desc_, x_desc_, bwd_data_pref, workspace_byte_limit_, &bp_data_alg_)); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/tool/docker/devel/conda/cuda/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/conda/cuda/Dockerfile b/tool/docker/devel/conda/cuda/Dockerfile index ea360f3..4b62271 100644 --- a/tool/docker/devel/conda/cuda/Dockerfile +++ b/tool/docker/devel/conda/cuda/Dockerfile @@ -14,13 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Base unbuntu 14.04 image from nvidia/cuda + # Change tags to build with different cuda/cudnn versions: FROM nvidia/cuda:9.0-devel-ubuntu16.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 # install dependencies RUN apt-get update \ @@ -37,12 +33,6 @@ RUN /root/miniconda/bin/conda install conda-build RUN /root/miniconda/bin/conda install anaconda-client ENV PATH /root/miniconda/bin:${PATH} -# set environment -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git # config ssh service RUN mkdir /var/run/sshd http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/tool/docker/devel/native/ubuntu/cuda/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/ubuntu/cuda/Dockerfile b/tool/docker/devel/native/ubuntu/cuda/Dockerfile deleted file mode 100644 index 03d53c0..0000000 --- a/tool/docker/devel/native/ubuntu/cuda/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# Change tags to build with different cuda/cudnn versions: -FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server swig\ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf future - - -# install swig > 3.0.10 for ubuntu < 16.04 -# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ -# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ -# ./configure && make && make install - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -# for ubuntu 14.04 -# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# for ubuntu 16.04 prohibit -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/tool/docker/devel/native/ubuntu/cuda/py2/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/ubuntu/cuda/py2/Dockerfile b/tool/docker/devel/native/ubuntu/cuda/py2/Dockerfile new file mode 100644 index 0000000..8e6c0d0 --- /dev/null +++ b/tool/docker/devel/native/ubuntu/cuda/py2/Dockerfile @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Change tags to build with different cuda/cudnn versions: +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev protobuf-compiler wget openssh-server swig python-dev python-pip python-setuptools\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf future + + +# install swig > 3.0.10 for ubuntu < 16.04 +# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ +# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ +# ./configure && make && make install + +# set environment +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +# for ubuntu 14.04 +# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# for ubuntu 16.04 prohibit +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile b/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile new file mode 100644 index 0000000..2ecd3a9 --- /dev/null +++ b/tool/docker/devel/native/ubuntu/cuda/py3/Dockerfile @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Change tags to build with different cuda/cudnn versions: +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev protobuf-compiler wget openssh-server swig python3-dev python3-pip python3-setuptools\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip3 install -U pip3 wheel numpy setuptools unittest-xml-reporting protobuf future + + +# install swig > 3.0.10 for ubuntu < 16.04 +# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ +# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ +# ./configure && make && make install + +# set environment +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +# for ubuntu 14.04 +# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# for ubuntu 16.04 prohibit +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4ccb72ee/tool/jenkins/test.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/test.sh b/tool/jenkins/test.sh index 765fde1..039c15a 100644 --- a/tool/jenkins/test.sh +++ b/tool/jenkins/test.sh @@ -24,29 +24,24 @@ echo Compile and test Singa... echo workspace: `pwd` echo OS version: `cat /etc/issue` echo kernal version: `uname -a` +echo parameters: $1 COMMIT=`git rev-parse --short HEAD` echo COMMIT HASH: $COMMIT + # set parameters CUDA="OFF" -CUDNN="OFF" -if [ -z ${CUDNN_PATH+x} ]; then - CUDA="ON" - CUDNN="ON" - export CMAKE_PREFIX_PATH=$CUDNN_PATH:$CMAKE_PREFIX_PATH - export CMAKE_INCLUDE_PATH=$CUDNN_PATH/include:$CMAKE_INCLUDE_PATH - export CMAKE_LIBRARY_PATH=$CUDNN_PATH/lib64:$CMAKE_LIBRARY_PATH +if [ $1 = "CUDA" ]; then + CUDA="ON" fi -# setup env -rm -rf build -mkdir build - # TODO(wangwei) test python 3 according to env variable PY3K -if [ `uname` = "Darwin" ]; then - EXTRA_ARGS="-DPYTHON_LIBRARY=`python-config --prefix`/lib/libpython2.7.dylib -DPYTHON_INCLUDE_DIR=`python-config --prefix`/include/python2.7/" -fi +#if [ `uname` = "Darwin" ]; then +# EXTRA_ARGS="-DPYTHON_LIBRARY=`python-config --prefix`/lib/libpython2.7.dylib -DPYTHON_INCLUDE_DIR=`python-config --prefix`/include/python2.7/" +#fi +rm -rf build +mkdir build # compile c++ code cd build cmake -DUSE_CUDA=$CUDA -DENABLE_TEST=ON $EXTRA_ARGS ../
