SINGA-346 Update cudnn from V5 to V7 Update conda build scripts to include cudnn (.h and .so) in conda singa package.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ef5fdaf6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ef5fdaf6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ef5fdaf6 Branch: refs/heads/master Commit: ef5fdaf652b1ac82e0edf0c07fc0499d13238581 Parents: c9f9471 Author: Wang Wei <[email protected]> Authored: Sun Apr 15 23:29:08 2018 +0800 Committer: Wang Wei <[email protected]> Committed: Wed Apr 18 22:36:40 2018 +0800 ---------------------------------------------------------------------- tool/conda/README.md | 29 ++++++++ tool/conda/build.sh | 17 +++-- tool/conda/meta.yaml | 9 ++- tool/jenkins/docker/devel/centos6/Dockerfile | 64 ------------------ .../docker/devel/native/centos6/Dockerfile | 64 ++++++++++++++++++ .../docker/devel/native/ubuntu/Dockerfile | 70 ++++++++++++++++++++ tool/jenkins/docker/devel/ubuntu/Dockerfile | 70 -------------------- 7 files changed, 178 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/README.md ---------------------------------------------------------------------- diff --git a/tool/conda/README.md b/tool/conda/README.md new file mode 100644 index 0000000..d30cf84 --- /dev/null +++ b/tool/conda/README.md @@ -0,0 +1,29 @@ +# Package SINGA using conda-build + +[conda-build](https://conda.io/docs/user-guide/tasks/build-packages/index.html) is a packaging tool like apt-get, which is associated with [anaconda cloud](https://anaconda.org/) for package management for both python and cpp libraries. + + +## Environment variables + +Depending on the building evironment, export the corresponding build string. + + # for singa with gpu, e.g. cuda8.0-cudnn7.0.5 + export BUILD_STR=cudax.y-cudnna.b.c + + # for singa running only on cpu + export BUILD_STR=cpu + + +To package SINGA with CUDA and CUDNN, + + export CUDNN_PATH=<path to cudnn folder> + +this folder should include a subfolder `include/cudnn.h` for the header file, and another subfolder `lib64` for the shared libraries. The BUILD_STR and CUDNN_PATH must be consistent. For example, if CUDNN_PATH is set, then BUILD_STR must be like cudax.y-cudnna.b.c. + +## Instruction + +After exporting the environment variables, execute the following command to compile SINGA and package it + + conda-build . + +You will see the package path from the screen output. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/build.sh b/tool/conda/build.sh index 6b3f262..2027725 100644 --- a/tool/conda/build.sh +++ b/tool/conda/build.sh @@ -16,19 +16,24 @@ # # to compile swig api files which depdend on numpy.i -export export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH +export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH # to let cmake use the dependent libs installed by conda, including python export CMAKE_PREFIX_PATH=$PREFIX -export CMAKE_INCLUDE_PATH=$SINGA_INCLUDE_PATH -export CMAKE_LIBRARY_PATH=$SINGA_LIBRARY_PATH +export CMAKE_INCLUDE_PATH=$PREFIX/include +export CMAKE_LIBRARY_PATH=$PREFIX/lib mkdir build cd build -USE_CUDA=ON -# singa with cuda and cudnn has the name as : singa-cudaxx-cudnnxx -if [ "$PKG_NAME" == "singa" ]; then USE_CUDA=OFF; fi +USE_CUDA=OFF +if [ -z ${CUDNN_PATH+x} ]; then + USE_CUDA=ON + cp $CUDNN_PATH/include $PREFIX/include + cp -P $CUDNN_PATH/lib64/* $PREFIX/lib/ +fi + PYTHON3=OFF +# PY3K is set by conda if [ "$PY3K" == "True" ]; then PYTHON3=ON; fi cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DPYTHON3=$PYTHON3 .. make http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/conda/meta.yaml ---------------------------------------------------------------------- diff --git a/tool/conda/meta.yaml b/tool/conda/meta.yaml index 5ff87a9..de542cd 100644 --- a/tool/conda/meta.yaml +++ b/tool/conda/meta.yaml @@ -1,16 +1,15 @@ package: - name: {{ environ.get('SINGA_NAME', 'singa') }} + name: singa version: "{{ GIT_DESCRIBE_TAG }}" source: git_url: https://github.com/apache/incubator-singa.git build: - number: {{ GIT_DESCRIBE_NUMBER }} + number: 0 + string: py{{py}}_{{ environ.get('BUILD_STR', 'cpu') }} script_env: - - CONDA_BLD_PATH - - SINGA_INCLUDE_PATH - - SINGA_LIBRARY_PATH + - CUDNN_PATH requirements: build: http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/centos6/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/centos6/Dockerfile b/tool/jenkins/docker/devel/centos6/Dockerfile deleted file mode 100644 index 61b30f6..0000000 --- a/tool/jenkins/docker/devel/centos6/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Base unbuntu 14.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -FROM nvidia/cuda:7.5-cudnn5-devel-centos6 - -# install dependencies -RUN yum -y update && yum -y install git wget openssh-server cmake - - -# install conda, conda-build and anaconda-client -RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; -RUN bash miniconda.sh -b -p /root/miniconda -ENV PATH /root/miniconda/bin:${PATH} -RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no -RUN /root/miniconda/bin/conda update -q conda -RUN /root/miniconda/bin/conda install conda-build -RUN /root/miniconda/bin/conda install anaconda-client - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key - -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config -RUN echo 'root:singa' | chpasswd - -#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/native/centos6/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/native/centos6/Dockerfile b/tool/jenkins/docker/devel/native/centos6/Dockerfile new file mode 100644 index 0000000..61b30f6 --- /dev/null +++ b/tool/jenkins/docker/devel/native/centos6/Dockerfile @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 +FROM nvidia/cuda:7.5-cudnn5-devel-centos6 + +# install dependencies +RUN yum -y update && yum -y install git wget openssh-server cmake + + +# install conda, conda-build and anaconda-client +RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +RUN bash miniconda.sh -b -p /root/miniconda +ENV PATH /root/miniconda/bin:${PATH} +RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no +RUN /root/miniconda/bin/conda update -q conda +RUN /root/miniconda/bin/conda install conda-build +RUN /root/miniconda/bin/conda install anaconda-client + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# download singa source +# RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key +RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key + +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config +RUN echo 'root:singa' | chpasswd + +#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/native/ubuntu/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/native/ubuntu/Dockerfile b/tool/jenkins/docker/devel/native/ubuntu/Dockerfile new file mode 100644 index 0000000..302c579 --- /dev/null +++ b/tool/jenkins/docker/devel/native/ubuntu/Dockerfile @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server swig\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf future + + +# install swig > 3.0.10 for ubuntu < 16.04 +# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ +# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ +# ./configure && make && make install + +# install conda, conda-build and anaconda-client +RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +RUN bash miniconda.sh -b -p /root/miniconda +RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no +RUN /root/miniconda/bin/conda update -q conda +RUN /root/miniconda/bin/conda install conda-build +RUN /root/miniconda/bin/conda install anaconda-client +ENV PATH /root/miniconda/bin:${PATH} + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# download singa source +# RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +# for ubuntu 14.04 +# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# for ubuntu 16.04 prohibit +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef5fdaf6/tool/jenkins/docker/devel/ubuntu/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/ubuntu/Dockerfile b/tool/jenkins/docker/devel/ubuntu/Dockerfile deleted file mode 100644 index 302c579..0000000 --- a/tool/jenkins/docker/devel/ubuntu/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Base unbuntu 14.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server swig\ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf future - - -# install swig > 3.0.10 for ubuntu < 16.04 -# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ -# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ -# ./configure && make && make install - -# install conda, conda-build and anaconda-client -RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; -RUN bash miniconda.sh -b -p /root/miniconda -RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no -RUN /root/miniconda/bin/conda update -q conda -RUN /root/miniconda/bin/conda install conda-build -RUN /root/miniconda/bin/conda install anaconda-client -ENV PATH /root/miniconda/bin:${PATH} - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -# for ubuntu 14.04 -# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# for ubuntu 16.04 prohibit -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"]
