SINGA-303 Create conda packages Combine with travis: 1. singa at travis/apache does normal building and test (apt-get and homebrew) 2. singa at travis/nusdbsystem creates conda packages for linux and osx, and upload them to anaconda (channel: nusdbsystem)
Create GPU conda packages using nvidi-docker. package name is like singa-cudaxx-cudnnyy Update cmakefile to disable test by default TODO: update jenkin's setting to create conda GPU packages, and replace wheel packages which may result in runtime linking errors, e.g., on osx NOTE: 1. conda's protobuf (3.0.0) was compiled using gcc 4; if we compile singa using gcc 5 (abi != 0), then there will be linking error; Therefore, we are using gcc4.8 to compile singa. ubuntu14.04 comes with gcc 4.8 and ubuntu16.04 comes with gcc 5. 2. use -stdlib=libc++ is necessary to for the cxxflag in osx 3. nvidia-docker puts cudnn into different system folders for diff docker images. cmake should be able to find them, since they are in system paths. hence, no need to export cmake paths. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3417dd9d Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3417dd9d Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3417dd9d Branch: refs/heads/master Commit: 3417dd9d638331e2451e767c05b12fff20484a8c Parents: 5d17266 Author: Wei Wang <[email protected]> Authored: Wed Feb 22 16:51:14 2017 +0800 Committer: Wei Wang <[email protected]> Committed: Thu Feb 23 14:47:23 2017 +0800 ---------------------------------------------------------------------- .travis.yml | 2 +- CMakeLists.txt | 17 ++++++++++++----- tool/conda/build.sh | 25 +++++++++++++++++++++---- tool/conda/meta.yaml | 27 +++++++++++++++++++-------- tool/travis/build.sh | 4 ++-- tool/travis/conda.sh | 12 +++++++++++- tool/travis/depends.sh | 3 +++ 7 files changed, 69 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index c8d98fb..3c1fab4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: include: - os: osx compiler: clang - osx_image: xcode7.3 + osx_image: xcode8 - os: linux dist: trusty compiler: gcc http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index d7acd09..1fbe30c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,10 +29,13 @@ LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #message(STATUS "module path: ${CMAKE_MODULE_PATH}") # Flags -IF(UNIX OR APPLE) - SET(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -std=c++11 -g -O2 -fPIC -Wall -pthread") +IF(UNIX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -O2 -fPIC -Wall -pthread") ENDIF() +IF(APPLE) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -g -O2 -fPIC -Wall -pthread") +ENDIF() + IF(WIN32) IF (MSVC) MESSAGE(STATUS "Using msvc compiler") @@ -57,6 +60,7 @@ OPTION(USE_PYTHON "Generate py wrappers" ON) OPTION(USE_JAVA "Generate java wrappers" OFF) OPTION(USE_OPENCL "Use OpenCL" OFF) OPTION(ENABLE_DIST "Enable distributed training" OFF) +OPTION(ENABLE_TEST "Enable unit test" OFF) OPTION(DISABLE_WARNINGS "Disable warnings under windows" ON) OPTION(USE_MODULES "Compile dependent libs as submodules together with singa" OFF) #OPTION(USE_SHARED_LIBS "Use shared library" OFF) @@ -145,8 +149,11 @@ ENDIF() INCLUDE_DIRECTORIES("${CMAKE_BINARY_DIR}/include") ADD_SUBDIRECTORY(src) -ADD_SUBDIRECTORY(test) -ADD_SUBDIRECTORY(examples) + +IF (ENABLE_TEST) + ADD_SUBDIRECTORY(test) + ADD_SUBDIRECTORY(examples) +ENDIF() IF (USE_PYTHON) ADD_SUBDIRECTORY(python) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/tool/conda/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/build.sh b/tool/conda/build.sh index 414bd49..f3a84d9 100644 --- a/tool/conda/build.sh +++ b/tool/conda/build.sh @@ -1,10 +1,27 @@ -export CMAKE_PREFIX_PATH=$PREFIX -export export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -echo $PREFIX +export export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX=$PREFIX .. +USE_CUDA=ON +# singa with cuda and cudnn has the name as : singa-cudaxx-cudnnxx +if [ "$PKG_NAME" == "singa" ]; then USE_CUDA=OFF; fi +cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA .. make make install http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/tool/conda/meta.yaml ---------------------------------------------------------------------- diff --git a/tool/conda/meta.yaml b/tool/conda/meta.yaml index 95fbc58..c22b64f 100644 --- a/tool/conda/meta.yaml +++ b/tool/conda/meta.yaml @@ -1,13 +1,15 @@ package: - name: singa - version: "1.1.0" + name: {{ environ.get('SINGA_NAME', 'singa') }} + version: "{{ GIT_DESCRIBE_TAG }}" source: - git_url: https://github.com/nudles/incubator-singa.git + git_url: https://github.com/nusdbsystem/singa-osx.git build: number: {{ GIT_DESCRIBE_NUMBER }} + script_env: + - CONDA_BLD_PATH requirements: build: @@ -15,21 +17,30 @@ requirements: - numpy >=1.10 - swig >=3.0 - openblas >=0.2.10 - - libprotobuf >=2.5,<2.7 - - glog # [osx] + - libprotobuf 3.0.0 + - glog 0.3.4 run: - python 2.7* - numpy >=1.10 - - libprotobuf >=2.5,<3.0 + - libprotobuf 3.0.0 + - protobuf 3.0.0 + - glog 0.3.4 - openblas >=0.2.10 - flask >=0.10.1 - - flask_cors >=3.0.2 + - flask-cors >=3.0.2 - pillow >=2.3.0 + +test: + source_files: + - test/python/*.py + requires: - unittest-xml-reporting - - glog # [osx] + test: + - python run.py about: home: http://singa.apache.org/ license: Apache V2 license_file: LICENSE + summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/tool/travis/build.sh ---------------------------------------------------------------------- diff --git a/tool/travis/build.sh b/tool/travis/build.sh index 3f13bce..b1b8851 100644 --- a/tool/travis/build.sh +++ b/tool/travis/build.sh @@ -24,9 +24,9 @@ then export CMAKE_INCLUDE_PATH=/usr/local/opt/openblas/include:$CMAKE_INCLUDE_PATH; fi mkdir build && cd build; - cmake -DUSE_CUDA=OFF -DUSE_PYTHON=OFF ..; + cmake -DUSE_CUDA=OFF -DUSE_PYTHON=OFF -DENABLE_TEST=ON ..; make; ./bin/test_singa --gtest_output=xml:./../gtest.xml; else - bash tool/travis/conda.sh; + bash -e tool/travis/conda.sh; fi http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/tool/travis/conda.sh ---------------------------------------------------------------------- diff --git a/tool/travis/conda.sh b/tool/travis/conda.sh index 6f961eb..f4b853b 100644 --- a/tool/travis/conda.sh +++ b/tool/travis/conda.sh @@ -17,12 +17,22 @@ # to build SINGA package and upload it to anaconda +set -x + +# anaconda login user name USER=nusdbsystem OS=$TRAVIS_OS_NAME-64 -mkdir ~/conda-bld +export PATH="$HOME/miniconda/bin:$PATH" conda config --set anaconda_upload no + +# save the package at given folder, then we can upload using singa-*.tar.bz2 suffix=`TZ=Asia/Singapore date +%Y-%m-%d-%H-%M-%S` export CONDA_BLD_PATH=~/conda-bld-$suffix +mkdir $CONDA_BLD_PATH + conda build tool/conda/ + +# turn off debug to hide the token in travis log +set +x anaconda -t $ANACONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/singa-*.tar.bz2 --force http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3417dd9d/tool/travis/depends.sh ---------------------------------------------------------------------- diff --git a/tool/travis/depends.sh b/tool/travis/depends.sh index 8702c52..020b046 100644 --- a/tool/travis/depends.sh +++ b/tool/travis/depends.sh @@ -15,6 +15,8 @@ # limitations under the License. # +# if no env var (i.e., token), then do normal build and test; +# otherwise use conda to build and package if [[ "$TRAVIS_SECURE_ENV_VARS" == "false" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; @@ -27,6 +29,7 @@ then brew install openblas protobuf260; fi else + # install miniconda if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
