SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
update python unittest to genreate xml report; update dockerfile to install unittest-xml-reporting; update jenkins script to run python unittest; add USE_MODULES for compiling protobuf and blas skip cuda test if pysinga is not compiled with cuda Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/e44658aa Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/e44658aa Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/e44658aa Branch: refs/heads/master Commit: e44658aa763f9aef9a38c3ea57855e71dd798ef8 Parents: 3f23c0d Author: wangwei <[email protected]> Authored: Thu Dec 15 17:56:23 2016 +0800 Committer: wangwei <[email protected]> Committed: Sat Dec 24 09:33:27 2016 +0800 ---------------------------------------------------------------------- python/setup.py.in | 9 +++++---- test/python/run.py | 9 +++++++-- test/python/test_optimizer.py | 9 ++++++--- tool/jenkins/README.md | 2 +- tool/jenkins/docker/ubuntu14.04/Dockerfile | 3 +-- tool/jenkins/docker/ubuntu16.04/Dockerfile | 5 ++--- tool/jenkins/jenkins_wheel.sh | 13 ++++++++----- 7 files changed, 30 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/python/setup.py.in ---------------------------------------------------------------------- diff --git a/python/setup.py.in b/python/setup.py.in index 95550d8..503ac8a 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -6,15 +6,15 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# # Always prefer setuptools over distutils from setuptools import setup @@ -60,7 +60,8 @@ setup( install_requires=[ 'numpy>=1.11.0', - 'protobuf>=2.5.0,<3' + 'protobuf>=2.5.0,<3', + 'unittest-xml-reporting' ], #List additional groups of dependencies here (e.g. development http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/test/python/run.py ---------------------------------------------------------------------- diff --git a/test/python/run.py b/test/python/run.py index ae33fbd..8f7a45f 100644 --- a/test/python/run.py +++ b/test/python/run.py @@ -17,8 +17,13 @@ # import unittest +import xmlrunner loader = unittest.TestLoader() tests = loader.discover('.') -testRunner = unittest.runner.TextTestRunner() -testRunner.run(tests) +# testRunner = unittest.runner.TextTestRunner() +with open('unittest.xml', 'wb') as output: + testRunner = xmlrunner.XMLTestRunner(output=output, + failfast=False, + buffer=False) + testRunner.run(tests) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/test/python/test_optimizer.py ---------------------------------------------------------------------- diff --git a/test/python/test_optimizer.py b/test/python/test_optimizer.py index bb3613d..601aada 100644 --- a/test/python/test_optimizer.py +++ b/test/python/test_optimizer.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. # ============================================================================= -import sys -import os import unittest import numpy as np @@ -24,8 +22,10 @@ import numpy as np import singa.tensor as tensor import singa.optimizer as opt import singa.device as device +from singa import singa_wrap -cuda = device.create_cuda_gpu() +if singa_wrap.USE_CUDA: + cuda = device.create_cuda_gpu() class TestOptimizer(unittest.TestCase): @@ -48,6 +48,7 @@ class TestOptimizer(unittest.TestCase): for i in range(self.W.size()): self.assertAlmostEqual(w[i], self.np_W[i] - lr * self.np_g[i]) + @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled') def test_sgd_cuda(self): lr = 0.1 sgd = opt.SGD(lr) @@ -67,6 +68,7 @@ class TestOptimizer(unittest.TestCase): for i in range(g.size): self.assertAlmostEqual(g[i], self.np_g[i] * threshold / nrm) + @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled') def test_constraint_cuda(self): threshold = 0.02 self.to_cuda() @@ -87,6 +89,7 @@ class TestOptimizer(unittest.TestCase): self.assertAlmostEqual(g[i], self.np_g[i] + coefficient * self.np_W[i]) + @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled') def test_regularizer_cuda(self): coefficient = 0.0001 reg = opt.L2Regularizer(coefficient) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index 6bd90df..d20470b 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -38,7 +38,7 @@ Create a multi-configuration project and configure project as follows: ### Post-build Actions * Archive the artifacts - ``build/python/dist/**.whl`` - * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml`` + * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml`` ## Docker Images We provide in `docker` a number of singa docker images for Jenkins to use as http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/docker/ubuntu14.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile index b664689..9b974d2 100644 --- a/tool/jenkins/docker/ubuntu14.04/Dockerfile +++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile @@ -9,9 +9,8 @@ FROM nvidia/cuda:8.0-cudnn5-devel # install dependencies RUN apt-get update && apt-get install -y \ git vim wget build-essential autoconf libtool cmake \ - libprotobuf-dev libopenblas-dev protobuf-compiler \ libpcre3-dev python-dev python-pip && \ - pip install numpy + pip install numpy protobuf unittest-xml-reporting # install swig 3.0 RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/docker/ubuntu16.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile index e7835f4..612dd8b 100644 --- a/tool/jenkins/docker/ubuntu16.04/Dockerfile +++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile @@ -5,10 +5,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 # install dependencies RUN apt-get update && apt-get install -y \ - git vim wget build-essential autoconf libtool cmake \ - libprotobuf-dev libopenblas-dev protobuf-compiler \ + git wget build-essential autoconf libtool cmake \ libpcre3-dev python-dev python-pip swig && \ - pip install numpy + pip install numpy protobuf unittest-xml-reporting # set environment ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/jenkins_wheel.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh index f82d070..c3d075f 100644 --- a/tool/jenkins/jenkins_wheel.sh +++ b/tool/jenkins/jenkins_wheel.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash #/** # * # * Licensed to the Apache Software Foundation (ASF) under one @@ -26,7 +26,7 @@ echo workspace: `pwd` echo OS version: `cat /etc/issue` echo kernal version: `uname -a` echo CUDA version: $CUDA_VERSION -echo CUDNN version: $CUDNN_VERSION +echo CUDNN version: $CUDNN_VERSION # set parameters CUDNN="OFF" if [ $1 = "CUDNN" ]; then @@ -37,14 +37,17 @@ rm -rf build mkdir build # compile singa c++ cd build -cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN ../ +cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../ make -# unit test -./bin/test_singa --gtest_output=xml:./../gtest.xml +# unit test cpp code +./bin/test_singa --gtest_output=xml:./gtest.xml # compile pysinga cd python python setup.py bdist_wheel # rename dist cd dist mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl +# unit test python code +cd ../../../test/python +PYTHONPATH=../../build/python/ python run.py echo Job finished...
