Repository: incubator-singa Updated Branches: refs/heads/master b4ea650ef -> 163452e74
SINGA-345 Update Jenkins and fix bugs in compliation update jenkins README with more details about the Docker build agent; fix a bug in cifar10 CMakefile.txt due to the change of file name: alexnet->cnn install future for py3 for jenkins docker build agent Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/95b535d6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/95b535d6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/95b535d6 Branch: refs/heads/master Commit: 95b535d63a59b6b43b4f72898cc661c38f69b0f2 Parents: b4ea650 Author: wang wei <[email protected]> Authored: Thu Apr 5 22:26:46 2018 +0700 Committer: wang wei <[email protected]> Committed: Fri Apr 6 08:57:10 2018 +0700 ---------------------------------------------------------------------- examples/cifar10/CMakeLists.txt | 14 ++-- tool/jenkins/README.md | 25 ++++++- tool/jenkins/docker/devel/ubuntu/Dockerfile | 70 ++++++++++++++++++++ .../jenkins/docker/devel/ubuntu14.04/Dockerfile | 67 ------------------- 4 files changed, 100 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/95b535d6/examples/cifar10/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/examples/cifar10/CMakeLists.txt b/examples/cifar10/CMakeLists.txt index 8026467..3fc4e57 100644 --- a/examples/cifar10/CMakeLists.txt +++ b/examples/cifar10/CMakeLists.txt @@ -20,16 +20,16 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) -ADD_EXECUTABLE(alexnet alexnet.cc) -ADD_DEPENDENCIES(alexnet singa) -TARGET_LINK_LIBRARIES(alexnet singa) +ADD_EXECUTABLE(cnn cnn.cc) +ADD_DEPENDENCIES(cnn singa) +TARGET_LINK_LIBRARIES(cnn singa) IF(USE_CUDNN) -ADD_EXECUTABLE(alexnet-parallel alexnet-parallel.cc) -ADD_DEPENDENCIES(alexnet-parallel singa) -TARGET_LINK_LIBRARIES(alexnet-parallel singa) -#SET_TARGET_PROPERTIES(alexnet-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") +ADD_EXECUTABLE(cnn-parallel cnn-parallel.cc) +ADD_DEPENDENCIES(cnn-parallel singa) +TARGET_LINK_LIBRARIES(cnn-parallel singa) +#SET_TARGET_PROPERTIES(cnn-parallel PROPERTIES LINK_FLAGS "${LINK_FLAGS} -pthread") ADD_EXECUTABLE(vgg-parallel vgg-parallel.cc) ADD_DEPENDENCIES(vgg-parallel singa) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/95b535d6/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index b734b8f..6228c88 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -11,6 +11,10 @@ Continuous integration for CPU systems is enabled via [Travis](../travis). [Jenkins Official Wiki](https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins) The slave nodes for running different building environments are configured under 'Manage Jenkins'->'Manage nodes'. +Change Jenkins time zone by executing the following code in 'Mange jenkins' -> 'Script Console': + + System.setProperty('org.apache.commons.jelly.tags.fmt.timeZone', 'Asia/Singapore') + ## Configure Jenkins for Unit Testing and Binary Package Generation Create a multi-configuration project and configure project as follows: @@ -88,10 +92,27 @@ Each node should configure the following environment variable debian/32/84d56b7/ubuntu14.04-cuda8.0-cudnn5/singa-1.0.1.deb ### Docker Images -We provide in a number of singa docker [images](./docker) for Jenkins to use as slaves. +We provide a number of singa docker [images](./docker) for Jenkins to use as slaves. To run the docker images, - nvidia-docker run --name <jenkins-slaveXX> -d <Image ID> + nvidia-docker run --name <jenkins-slaveXX> -d <Image ID> -P + +To add the container into a network for easy access + + docker network create <network name> + docker network connect <network name> <jenkins-slaveXX> + +After connecting both the jenkins and slave contaniners into the same network, we can ssh to the slave from jenkins container like + + + # inside jenkins container + ssh root@<jenkins-slaveXX> + +You need execute the above command manually for the first ssh login + +In the Jenkins node configuration page, the container name is used to configure the `Host` field. +Notice that Oracle username and account are required to luanch the node by Jenkins. + ## Configure Jenkins for SINGA Website Updates http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/95b535d6/tool/jenkins/docker/devel/ubuntu/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/ubuntu/Dockerfile b/tool/jenkins/docker/devel/ubuntu/Dockerfile new file mode 100644 index 0000000..302c579 --- /dev/null +++ b/tool/jenkins/docker/devel/ubuntu/Dockerfile @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 +# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server swig\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf future + + +# install swig > 3.0.10 for ubuntu < 16.04 +# RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ +# tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ +# ./configure && make && make install + +# install conda, conda-build and anaconda-client +RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +RUN bash miniconda.sh -b -p /root/miniconda +RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no +RUN /root/miniconda/bin/conda update -q conda +RUN /root/miniconda/bin/conda install conda-build +RUN /root/miniconda/bin/conda install anaconda-client +ENV PATH /root/miniconda/bin:${PATH} + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} + +# download singa source +# RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +# for ubuntu 14.04 +# RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# for ubuntu 16.04 prohibit +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/95b535d6/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile b/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile deleted file mode 100644 index f907e07..0000000 --- a/tool/jenkins/docker/devel/ubuntu14.04/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Base unbuntu 14.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -# FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 -# FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04 -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server \ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf - - -# install swig 3.0 -RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ - tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ - ./configure && make && make install - -# install conda, conda-build and anaconda-client -RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; -RUN bash miniconda.sh -b -p /root/miniconda -RUN /root/miniconda/bin/conda config --set always_yes yes --set changeps1 no -RUN /root/miniconda/bin/conda update -q conda -RUN /root/miniconda/bin/conda install conda-build -RUN /root/miniconda/bin/conda install anaconda-client -ENV PATH /root/miniconda/bin:${PATH} - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} - -# download singa source -# RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"]
