SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
Update the script to package the whl file into a folder, whose name indicates the version of OS, cuda and cudnn. Update the jenkins/docker files to avoid installing libprotobuf and openblas Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/761c463d Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/761c463d Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/761c463d Branch: refs/heads/master Commit: 761c463dd8a2aedd59af7f3a936e359080e6828d Parents: e44658a Author: wangwei <[email protected]> Authored: Fri Dec 23 15:16:19 2016 +0800 Committer: wangwei <[email protected]> Committed: Sat Dec 24 09:33:27 2016 +0800 ---------------------------------------------------------------------- tool/jenkins/README.md | 8 ++++---- tool/jenkins/docker/ubuntu14.04/Dockerfile | 11 +++++++---- tool/jenkins/docker/ubuntu16.04/Dockerfile | 17 +++++++++++------ tool/jenkins/jenkins_wheel.sh | 21 +++++++++++++++++---- 4 files changed, 39 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index d20470b..1b5fb7c 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -30,19 +30,19 @@ Create a multi-configuration project and configure project as follows: * Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes) ### Configuration Matrix - * User-defined Axis - name ``support`` values ``BLAS CUDNN`` + * User-defined Axis - name ``lang`` values ``CPP CUDA`` * Slave - name ``env`` Node/label ``tick all nodes`` ### Build - * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $support`` + * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang`` ### Post-build Actions * Archive the artifacts - ``build/python/dist/**.whl`` + * Send build artifacts over SSH - source files ``build/python/dist/*.tar.gz``, Remove prefix `build/python/dist`, Exec command `tar xf *.tar.gz && rm *.tar.gz` * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml`` ## Docker Images -We provide in `docker` a number of singa docker images for Jenkins to use as -slaves. +We provide in `docker` a number of singa docker images for Jenkins to use as slaves. ## Access Control Use `Role Strategy Plugin` to give read access for anonymous users. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu14.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile index 9b974d2..6d96983 100644 --- a/tool/jenkins/docker/ubuntu14.04/Dockerfile +++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile @@ -7,10 +7,12 @@ FROM nvidia/cuda:8.0-cudnn5-devel # install dependencies -RUN apt-get update && apt-get install -y \ - git vim wget build-essential autoconf libtool cmake \ - libpcre3-dev python-dev python-pip && \ - pip install numpy protobuf unittest-xml-reporting +RUN apt-get update \ + && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3 + # install swig 3.0 RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ @@ -21,6 +23,7 @@ RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} +ENV OS_VERSION ubuntu14.04 # download singa source RUN git clone https://github.com/apache/incubator-singa.git http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu16.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile index 612dd8b..1a9434c 100644 --- a/tool/jenkins/docker/ubuntu16.04/Dockerfile +++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile @@ -4,21 +4,26 @@ FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 # install dependencies -RUN apt-get update && apt-get install -y \ - git wget build-essential autoconf libtool cmake \ - libpcre3-dev python-dev python-pip swig && \ - pip install numpy protobuf unittest-xml-reporting + +MAINTAINER incubator-singa [email protected] + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3 # set environment ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} +ENV OS_VERSION ubuntu16.04 # download singa source RUN git clone https://github.com/apache/incubator-singa.git -# install ssh service -RUN apt-get install -y openssh-server +# config ssh service RUN mkdir /var/run/sshd RUN echo 'root:singa' | chpasswd RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/jenkins_wheel.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh index c3d075f..f131f68 100644 --- a/tool/jenkins/jenkins_wheel.sh +++ b/tool/jenkins/jenkins_wheel.sh @@ -27,26 +27,39 @@ echo OS version: `cat /etc/issue` echo kernal version: `uname -a` echo CUDA version: $CUDA_VERSION echo CUDNN version: $CUDNN_VERSION +echo OS name: $OS_VERSION +COMMIT=`git rev-parse HEAD` +echo COMMIT HASH: $COMMIT # set parameters +CUDA="OFF" CUDNN="OFF" -if [ $1 = "CUDNN" ]; then +FOLDER=$COMMIT/$OS_VERSION-cpp/ +if [ $1 = "CUDA" ]; then + CUDA="ON" CUDNN="ON" + FOLDER=$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION/ fi +echo wheel file folder: build/python/dist/whl/$FOLDER + # setup env rm -rf build mkdir build + # compile singa c++ cd build -cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../ +cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDA -DUSE_MODULES=ON -DUSE_MODULES=ON ../ make # unit test cpp code ./bin/test_singa --gtest_output=xml:./gtest.xml # compile pysinga cd python python setup.py bdist_wheel -# rename dist +# mv whl file to a folder whose name identifies the OS, CUDA, CUDNN etc. cd dist -mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl +mkdir -p $FOLDER +mv *.whl $FOLDER/ +tar czf $FOLDER.tar.gz $FOLDER/* + # unit test python code cd ../../../test/python PYTHONPATH=../../build/python/ python run.py
