SINGA-359 Create alias for conda packages Move conda build scripts into tool/conda/singa folder as conda-build expect to work in a flat folder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/00573e26 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/00573e26 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/00573e26 Branch: refs/heads/master Commit: 00573e26b6aafc1f3c8852bc3397c3a007cd2c89 Parents: 35b9fed Author: Wang Wei <[email protected]> Authored: Sat May 12 16:58:13 2018 +0800 Committer: Wang Wei <[email protected]> Committed: Sat May 12 16:58:13 2018 +0800 ---------------------------------------------------------------------- tool/conda/README.md | 34 ---------------------------- tool/conda/build.sh | 44 ------------------------------------ tool/conda/meta.yaml | 50 ----------------------------------------- tool/conda/singa/README.md | 34 ++++++++++++++++++++++++++++ tool/conda/singa/build.sh | 44 ++++++++++++++++++++++++++++++++++++ tool/conda/singa/meta.yaml | 50 +++++++++++++++++++++++++++++++++++++++++ tool/jenkins/README.md | 2 +- tool/travis/build.sh | 2 +- 8 files changed, 130 insertions(+), 130 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/README.md ---------------------------------------------------------------------- diff --git a/tool/conda/README.md b/tool/conda/README.md deleted file mode 100644 index 393330c..0000000 --- a/tool/conda/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Package Singa using conda-build - -[conda-build](https://conda.io/docs/user-guide/tasks/build-packages/index.html) is a packaging tool like apt-get, which is associated with [anaconda cloud](https://anaconda.org/) for package management for both python and cpp libraries. - - -## Environment variables - -Build string is a part of the conda package specification. We include the cuda and cudnn version in it if Singa is compiled with CUDA enabled. - - # for singa with gpu, e.g. cuda8.0-cudnn7.0.5 - export BUILD_STR=cudax.y-cudnna.b.c - - # for singa running only on cpu - export BUILD_STR=cpu - - -To package Singa with CUDA and CUDNN, - - export CUDNN_PATH=<path to cudnn folder> - -this folder should include a subfolder `include/cudnn.h` for the header file, and another subfolder `lib64` for the shared libraries. The BUILD_STR and CUDNN_PATH must be consistent. For example, if CUDNN_PATH is set, then BUILD_STR must be like cudax.y-cudnna.b.c. CUDNN must be provided if we want to compiled Singa with CUDA enabled. - -## Instruction - -After exporting the environment variables, execute the following command to compile Singa and package it - - conda-build . --python 3.6 (or 2.7) - -You will see the package path from the screen output. - -To clean the cache - - conda clean -ay - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/build.sh b/tool/conda/build.sh deleted file mode 100644 index 91a2f3b..0000000 --- a/tool/conda/build.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# to compile swig api files which depdend on numpy.i -# export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH - -# to let cmake use the dependent libs installed by conda, including python -export CMAKE_PREFIX_PATH=$PREFIX:$CMAKE_PREFIX_PATH -export CMAKE_INCLUDE_PATH=$PREFIX/include:$CMAKE_INCLUDE_PATH -export CMAKE_LIBRARY_PATH=$PREFIX/lib:$CMAKE_LIBRARY_PATH - - -if [ -z ${CUDNN_PATH+x} ]; then - USE_CUDA=OFF -else - USE_CUDA=ON - cp -r $CUDNN_PATH/include $PREFIX/include - cp -P $CUDNN_PATH/lib64/libcudnn.so* $PREFIX/lib/ -fi - -USE_PYTHON3=OFF -# PY3K is set by conda -if [ "$PY3K" == "1" ]; then USE_PYTHON3=ON; fi - - -mkdir build -cd build -cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DUSE_PYTHON3=$USE_PYTHON3 .. -make -make install http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/meta.yaml ---------------------------------------------------------------------- diff --git a/tool/conda/meta.yaml b/tool/conda/meta.yaml deleted file mode 100644 index de542cd..0000000 --- a/tool/conda/meta.yaml +++ /dev/null @@ -1,50 +0,0 @@ -package: - name: singa - version: "{{ GIT_DESCRIBE_TAG }}" - -source: - git_url: https://github.com/apache/incubator-singa.git - -build: - number: 0 - string: py{{py}}_{{ environ.get('BUILD_STR', 'cpu') }} - script_env: - - CUDNN_PATH - -requirements: - build: - - swig 3.0.10 - - openblas 0.2.19 - - protobuf 3.2.0 - - glog 0.3.4 - - libgfortran 3.0.0 # [osx] - - gcc 4.8.5 # [linux] - - python 2.7* [py27] - - python 3.6* [py36] - - numpy 1.12.0 - - run: - - openblas 0.2.19 - - protobuf 3.2.0 - - glog 0.3.4 - - libgfortran 3.0.0 # [osx] - - libgcc 4.8.5 # [linux] - - python 2.7* [py27] - - python 3.6* [py36] - - numpy >=1.12.0 - - flask >=0.10.1 - - flask-cors >=3.0.2 - - pillow >=2.3.0 - - future >=0.16.0 - -test: - source_files: - - test/python/*.py - test: - - python run.py - -about: - home: http://singa.apache.org/ - license: Apache V2 - license_file: LICENSE - summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/singa/README.md ---------------------------------------------------------------------- diff --git a/tool/conda/singa/README.md b/tool/conda/singa/README.md new file mode 100644 index 0000000..393330c --- /dev/null +++ b/tool/conda/singa/README.md @@ -0,0 +1,34 @@ +# Package Singa using conda-build + +[conda-build](https://conda.io/docs/user-guide/tasks/build-packages/index.html) is a packaging tool like apt-get, which is associated with [anaconda cloud](https://anaconda.org/) for package management for both python and cpp libraries. + + +## Environment variables + +Build string is a part of the conda package specification. We include the cuda and cudnn version in it if Singa is compiled with CUDA enabled. + + # for singa with gpu, e.g. cuda8.0-cudnn7.0.5 + export BUILD_STR=cudax.y-cudnna.b.c + + # for singa running only on cpu + export BUILD_STR=cpu + + +To package Singa with CUDA and CUDNN, + + export CUDNN_PATH=<path to cudnn folder> + +this folder should include a subfolder `include/cudnn.h` for the header file, and another subfolder `lib64` for the shared libraries. The BUILD_STR and CUDNN_PATH must be consistent. For example, if CUDNN_PATH is set, then BUILD_STR must be like cudax.y-cudnna.b.c. CUDNN must be provided if we want to compiled Singa with CUDA enabled. + +## Instruction + +After exporting the environment variables, execute the following command to compile Singa and package it + + conda-build . --python 3.6 (or 2.7) + +You will see the package path from the screen output. + +To clean the cache + + conda clean -ay + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/singa/build.sh ---------------------------------------------------------------------- diff --git a/tool/conda/singa/build.sh b/tool/conda/singa/build.sh new file mode 100644 index 0000000..91a2f3b --- /dev/null +++ b/tool/conda/singa/build.sh @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# to compile swig api files which depdend on numpy.i +# export CPLUS_INCLUDE_PATH=`python -c "from __future__ import print_function; import numpy; print(numpy.get_include())"`:$CPLUS_INCLUDE_PATH + +# to let cmake use the dependent libs installed by conda, including python +export CMAKE_PREFIX_PATH=$PREFIX:$CMAKE_PREFIX_PATH +export CMAKE_INCLUDE_PATH=$PREFIX/include:$CMAKE_INCLUDE_PATH +export CMAKE_LIBRARY_PATH=$PREFIX/lib:$CMAKE_LIBRARY_PATH + + +if [ -z ${CUDNN_PATH+x} ]; then + USE_CUDA=OFF +else + USE_CUDA=ON + cp -r $CUDNN_PATH/include $PREFIX/include + cp -P $CUDNN_PATH/lib64/libcudnn.so* $PREFIX/lib/ +fi + +USE_PYTHON3=OFF +# PY3K is set by conda +if [ "$PY3K" == "1" ]; then USE_PYTHON3=ON; fi + + +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -DUSE_CUDA=$USE_CUDA -DUSE_PYTHON3=$USE_PYTHON3 .. +make +make install http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/conda/singa/meta.yaml ---------------------------------------------------------------------- diff --git a/tool/conda/singa/meta.yaml b/tool/conda/singa/meta.yaml new file mode 100644 index 0000000..de542cd --- /dev/null +++ b/tool/conda/singa/meta.yaml @@ -0,0 +1,50 @@ +package: + name: singa + version: "{{ GIT_DESCRIBE_TAG }}" + +source: + git_url: https://github.com/apache/incubator-singa.git + +build: + number: 0 + string: py{{py}}_{{ environ.get('BUILD_STR', 'cpu') }} + script_env: + - CUDNN_PATH + +requirements: + build: + - swig 3.0.10 + - openblas 0.2.19 + - protobuf 3.2.0 + - glog 0.3.4 + - libgfortran 3.0.0 # [osx] + - gcc 4.8.5 # [linux] + - python 2.7* [py27] + - python 3.6* [py36] + - numpy 1.12.0 + + run: + - openblas 0.2.19 + - protobuf 3.2.0 + - glog 0.3.4 + - libgfortran 3.0.0 # [osx] + - libgcc 4.8.5 # [linux] + - python 2.7* [py27] + - python 3.6* [py36] + - numpy >=1.12.0 + - flask >=0.10.1 + - flask-cors >=3.0.2 + - pillow >=2.3.0 + - future >=0.16.0 + +test: + source_files: + - test/python/*.py + test: + - python run.py + +about: + home: http://singa.apache.org/ + license: Apache V2 + license_file: LICENSE + summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index 9114861..dc5ec56 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -54,7 +54,7 @@ The building script can do the following tasks: * create conda package and upload it to anaconda cloud Execute shell - command - /root/miniconda/bin/conda-build tool/conda + /root/miniconda/bin/conda-build tool/conda/singa /root/miniconda/bin/anaconda -t <ANACONDA_UPLOAD_TOKEN> upload -u nusdbsystem -l main /root/miniconda/linux-64/singa-*.so.*.tar.bz2 --force * (optional) create Debian package http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/00573e26/tool/travis/build.sh ---------------------------------------------------------------------- diff --git a/tool/travis/build.sh b/tool/travis/build.sh index 7ac3df8..f58d61c 100644 --- a/tool/travis/build.sh +++ b/tool/travis/build.sh @@ -30,7 +30,7 @@ suffix=$TRAVIS_JOB_NUMBER #`TZ=Asia/Singapore date +%Y-%m-%d-%H-%M-%S` export CONDA_BLD_PATH=~/conda-bld-$suffix mkdir $CONDA_BLD_PATH -conda build tool/conda/ --python 3.6 +conda build tool/conda/singa --python 3.6 conda install --use-local singa cd test/python $HOME/miniconda/bin/python run.py
