SINGA-310 Create conda packages on CentOS Add dockerfile for centos6 which is used to create conda package for old linux systems whose glibc is 2.14 Add gcc and libgcc as requirements for linux (not osx).
Add include and library path in conda build.sh for cudnn header and library update readme and repo link in meta.yml Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f5cb70c2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f5cb70c2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f5cb70c2 Branch: refs/heads/master Commit: f5cb70c21327a7abdf07986c067730b6284ee0fc Parents: 4fd44ed Author: Wei Wang <[email protected]> Authored: Tue Apr 11 14:22:47 2017 +0800 Committer: wangwei <[email protected]> Committed: Tue Apr 11 18:22:46 2017 +0800 ---------------------------------------------------------------------- tool/conda/build.sh | 2 + tool/conda/meta.yaml | 6 +- tool/jenkins/README.md | 1 + tool/jenkins/docker/devel/Dockerfile | 67 -------------------- tool/jenkins/docker/devel/centos6/Dockerfile | 64 +++++++++++++++++++ .../jenkins/docker/devel/ubuntu14.04/Dockerfile | 67 ++++++++++++++++++++ 6 files changed, 139 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/conda/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/build.sh b/tool/conda/build.sh index 926da71..2716452 100644 --- a/tool/conda/build.sh +++ b/tool/conda/build.sh @@ -20,6 +20,8 @@ export export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_inclu # to let cmake use the dependent libs installed by conda, including python export CMAKE_PREFIX_PATH=$PREFIX +export CMAKE_INCLUDE_PATH=$SINGA_INCLUDE_PATH +export CMAKE_LIBRARY_PATH=$SINGA_LIBRARY_PATH mkdir build cd build http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/conda/meta.yaml ---------------------------------------------------------------------- diff --git a/tool/conda/meta.yaml b/tool/conda/meta.yaml index a322f3a..67aeec7 100644 --- a/tool/conda/meta.yaml +++ b/tool/conda/meta.yaml @@ -3,13 +3,15 @@ package: version: "{{ GIT_DESCRIBE_TAG }}" source: - git_url: https://github.com/nusdbsystem/singa-osx.git + git_url: https://github.com/apache/incubator-singa.git build: number: {{ GIT_DESCRIBE_NUMBER }} script_env: - CONDA_BLD_PATH + - SINGA_INCLUDE_PATH + - SINGA_LIBRARY_PATH requirements: build: @@ -19,6 +21,7 @@ requirements: - openblas >=0.2.10 - protobuf 3.0.0 - glog 0.3.4 + - gcc 4.8.5 # [linux] run: - python 2.7* @@ -29,6 +32,7 @@ requirements: - flask >=0.10.1 - flask-cors >=3.0.2 - pillow >=2.3.0 + - libgcc 4.8.5 # [linux] test: source_files: http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index 9a76ee8..b734b8f 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -28,6 +28,7 @@ Each node should configure the following environment variable 3. ANACONDA_UPLOAD_TOKEN 4. SINGA_NAME=singa-cuda${CUDA_VERSION}-cudnn${CUDNN_VERSION} 5. OS_VERSION, e.g., ubuntu14.04 +6. SINGA_INCLUDE_PATH and SINGA_LIBRARY_PATH for the cudnn.h and libcudnn.so folder respectively ### General * Discard old builds - Max # of builds to keep - 50 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/jenkins/docker/devel/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/Dockerfile b/tool/jenkins/docker/devel/Dockerfile deleted file mode 100644 index 2ef191b..0000000 --- a/tool/jenkins/docker/devel/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Base unbuntu 14.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server \ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf - - -# install swig 3.0 -RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ - tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ - ./configure && make && make install - -# install conda, conda-build and anaconda-client -RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; -RUN bash miniconda.sh -b -p $HOME/miniconda -RUN $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no -RUN $HOME/miniconda/bin/conda update -q conda -RUN $HOME/miniconda/bin/conda install conda-build -RUN $HOME/miniconda/bin/conda install anaconda-client -ENV PATH $HOME/miniconda/bin:${PATH} - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/jenkins/docker/devel/centos6/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/centos6/Dockerfile b/tool/jenkins/docker/devel/centos6/Dockerfile new file mode 100644 index 0000000..61b30f6 --- /dev/null +++ b/tool/jenkins/docker/devel/centos6/Dockerfile @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 +FROM nvidia/cuda:7.5-cudnn5-devel-centos6 + +# install dependencies +RUN yum -y update && yum -y install git wget openssh-server cmake + + +# install conda, conda-build and anaconda-client +RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +RUN bash miniconda.sh -b -p /root/miniconda +ENV PATH /root/miniconda/bin:${PATH} +RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no +RUN /root/miniconda/bin/conda update -q conda +RUN /root/miniconda/bin/conda install conda-build +RUN /root/miniconda/bin/conda install anaconda-client + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# download singa source +# RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key +RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key + +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config +RUN echo 'root:singa' | chpasswd + +#RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f5cb70c2/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile b/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile new file mode 100644 index 0000000..f907e07 --- /dev/null +++ b/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server \ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf + + +# install swig 3.0 +RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ + tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ + ./configure && make && make install + +# install conda, conda-build and anaconda-client +RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +RUN bash miniconda.sh -b -p /root/miniconda +RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no +RUN /root/miniconda/bin/conda update -q conda +RUN /root/miniconda/bin/conda install conda-build +RUN /root/miniconda/bin/conda install anaconda-client +ENV PATH /root/miniconda/bin:${PATH} + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# download singa source +# RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"]
