Repository: incubator-singa Updated Branches: refs/heads/dev c50255259 -> 37e6ad283
SINGA-217 build python package with setup.py - modify CMakefiles to build a standard python singa package - remove all hard code path - after cmake and make, can run "python unittest.py" under "/test/python/", currently the "build" path is hard code in unittest.py - user can install singa package by command "sudo pip install ." under "build/python" directory (better run in python virltualenv) make sure libsinga is installed (make install). After this, user can remove hard code path in the unittest.py. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/37e6ad28 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/37e6ad28 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/37e6ad28 Branch: refs/heads/dev Commit: 37e6ad283ae8b095b1a675a8df3bdc40cfcbd544 Parents: c502552 Author: aaronwwf <[email protected]> Authored: Sat Jul 2 17:15:23 2016 +0800 Committer: aaronwwf <[email protected]> Committed: Sat Jul 2 20:46:44 2016 +0800 ---------------------------------------------------------------------- CMakeLists.txt | 28 +++++------- cmake/Protobuf.cmake | 6 +-- cmake/Utils.cmake | 6 ++- src/CMakeLists.txt | 16 ++++++- src/python/__init__.py | 3 ++ src/python/device.py | 6 +-- src/python/layer.py | 8 ++-- src/python/model.py | 2 +- src/python/setup.py.in | 76 ++++++++++++++++++++++++++++++++ src/python/tensor.py | 12 ++--- test/python/example_test_tensor.py | 78 ++++++++++++++++----------------- test/python/test_tensor.py | 14 +++--- 12 files changed, 164 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index a8be759..9eac5ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,13 +6,6 @@ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty) #message(STATUS "module path: ${CMAKE_MODULE_PATH}") -OPTION(USE_CBLAS "Use CBlas libs" ON) -OPTION(USE_CUDA "Use Cuda libs" ON) -OPTION(USE_CUDNN "Use Cudnn libs" ON) -OPTION(USE_OPENCV "Use opencv" OFF) -OPTION(USE_LMDB "Use LMDB libs" OFF) -OPTION(USE_PYTHON "Generate py wrappers" OFF) - # Flags IF(UNIX OR APPLE) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") @@ -22,14 +15,16 @@ IF(CMAKE_BUILD_TYPE=Debug) ENDIF() #message(STATUS "${CMAKE_CXX_FLAGS}") SET(SINGA_INCLUDE_DIR - "${CMAKE_SOURCE_DIR}/include;${PROJECT_BINARY_DIR}") -IF(USE_CUDA) - LIST(APPEND SINGA_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/lib/cnmem/include") -ELSEIF(USE_CUDA) - OPTION(USE_CUDNN "Use Cudnn libs" OFF) -ENDIF() + "${CMAKE_SOURCE_DIR}/include;${CMAKE_SOURCE_DIR}/lib/cnmem/include;${PROJECT_BINARY_DIR}") INCLUDE_DIRECTORIES(${SINGA_INCLUDE_DIR}) +OPTION(USE_CBLAS "Use CBlas libs" ON) +OPTION(USE_CUDA "Use Cuda libs" ON) +OPTION(USE_CUDNN "Use Cudnn libs" ON) +OPTION(USE_OPENCV "Use opencv" OFF) +OPTION(USE_LMDB "Use LMDB libs" OFF) +OPTION(USE_PYTHON "Generate py wrappers" ON) + INCLUDE("cmake/Dependencies.cmake") INCLUDE("cmake/Utils.cmake") ADD_DEFINITIONS(-DUSE_CMAKE) @@ -44,11 +39,8 @@ CONFIGURE_FILE ( SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) -IF(USE_CUDA) - ADD_SUBDIRECTORY(lib/cnmem) - LIST(APPEND SINGA_LINKER_LIBS cnmem) -ENDIF() - +ADD_SUBDIRECTORY(lib/cnmem) +LIST(APPEND SINGA_LINKER_LIBS cnmem) ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(test) ADD_SUBDIRECTORY(examples) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/cmake/Protobuf.cmake ---------------------------------------------------------------------- diff --git a/cmake/Protobuf.cmake b/cmake/Protobuf.cmake index c72b2c0..9324749 100644 --- a/cmake/Protobuf.cmake +++ b/cmake/Protobuf.cmake @@ -12,12 +12,12 @@ function(PROTOBUF_GENERATE_PYTHON OUTPUT) get_filename_component(FIL_WE ${FIL} NAME_WE) get_filename_component(PATH ${FIL} PATH) - list(APPEND ${OUTPUT} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py") + list(APPEND ${OUTPUT} "${CMAKE_BINARY_DIR}/python/singa/proto/${FIL_WE}_pb2.py") add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py" + OUTPUT "${CMAKE_BINARY_DIR}/python/singa/proto/${FIL_WE}_pb2.py" COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS --python_out ${CMAKE_CURRENT_BINARY_DIR} + ARGS --python_out ${CMAKE_BINARY_DIR}/python/singa/proto --proto_path ${PATH} ${ABS_FIL} DEPENDS ${ABS_FIL} COMMENT "Running Python protocol buffer compiler on ${FIL}" VERBATIM) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/cmake/Utils.cmake ---------------------------------------------------------------------- diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 7f9452e..c55cee3 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -3,11 +3,13 @@ macro(swig_generate_cxx pylist_variable) if(NOT EXISTS "${CMKAE_BINARY_DIR}/python") execute_process( COMMAND mkdir ${CMAKE_BINARY_DIR}/python + COMMAND mkdir ${CMAKE_BINARY_DIR}/python/singa + COMMAND mkdir ${CMAKE_BINARY_DIR}/python/singa/proto ERROR_QUIET) endif() execute_process( - COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include - -outdir ${CMAKE_BINARY_DIR}/python/ + COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include + -outdir ${CMAKE_BINARY_DIR}/python/singa ${ARGN}) set(${pylist_variable} "${CMAKE_SOURCE_DIR}/src/python/swig/singa_wrap.cxx") http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af09799..54d19ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,11 +81,25 @@ LIST(APPEND SINGA_LINKER_LIBS singa_io) IF(USE_PYTHON) FILE(GLOB python_files python/swig/singa.i) swig_generate_cxx(python_srcs ${python_files}) + FILE(COPY python/ DESTINATION ${CMAKE_BINARY_DIR}/python/singa FILES_MATCHING PATTERN "swig" EXCLUDE PATTERN "*.py") SET(python_cxxs "${CMAKE_SOURCE_DIR}/src/core/tensor/tensor.cc;${CMAKE_SOURCE_DIR}/src/core/device/device.cc") ADD_LIBRARY(_singa SHARED ${python_srcs} ${python_cxxs}) SET(WRAPPER_LINKER_LIBS "${SINGA_LINKER_LIBS};protobuf") TARGET_LINK_LIBRARIES(_singa ${WRAPPER_LINKER_LIBS}) TARGET_INCLUDE_DIRECTORIES(_singa PRIVATE ${PYTHON_INCLUDE_DIRS}) + #message(STATUS "PYTHON_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS}") + SET_TARGET_PROPERTIES(_singa - PROPERTIES PREFIX "") + PROPERTIES PREFIX "" + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa + ) + + #SETUP + SET(SETUP_PY_IN "python/setup.py.in") + set(SETUP_PY "${CMAKE_BINARY_DIR}/python/setup.py") + configure_file(${SETUP_PY_IN} ${SETUP_PY}) + + #COPY protobuf files to python/singa/proto + FILE(COPY ${CMAKE_BINARY_DIR}/python/singa/__init__.py DESTINATION ${CMAKE_BINARY_DIR}/python/singa/proto ) + ENDIF(USE_PYTHON) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/__init__.py ---------------------------------------------------------------------- diff --git a/src/python/__init__.py b/src/python/__init__.py new file mode 100644 index 0000000..c32dc40 --- /dev/null +++ b/src/python/__init__.py @@ -0,0 +1,3 @@ +def main(): + """Entry point for the application script""" + print("Welcome to SINGA!") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/device.py ---------------------------------------------------------------------- diff --git a/src/python/device.py b/src/python/device.py index 92a92e7..0877ae5 100644 --- a/src/python/device.py +++ b/src/python/device.py @@ -29,11 +29,7 @@ import sys import os import numpy as np -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/lib')) -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/python')) -import singa +from . import singa class Device(object): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/layer.py ---------------------------------------------------------------------- diff --git a/src/python/layer.py b/src/python/layer.py index b0bcfa9..a1ec556 100644 --- a/src/python/layer.py +++ b/src/python/layer.py @@ -24,12 +24,10 @@ import sys import os import numpy as np -import singa +from . import singa -sys.path.append(os.path.join(os.path.dirname(__file__), '../')) - -from core_pb2 import * -from model_pb2 import * +from .proto.core_pb2 import * +from .proto.model_pb2 import * class Layer(object): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/model.py ---------------------------------------------------------------------- diff --git a/src/python/model.py b/src/python/model.py index 6d9fe39..38d9950 100644 --- a/src/python/model.py +++ b/src/python/model.py @@ -17,5 +17,5 @@ # */ class Model(Object): - + pass http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/setup.py.in ---------------------------------------------------------------------- diff --git a/src/python/setup.py.in b/src/python/setup.py.in new file mode 100644 index 0000000..9002c70 --- /dev/null +++ b/src/python/setup.py.in @@ -0,0 +1,76 @@ +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from codecs import open +from os import path + +setup( + name='singa', + + version='${PACKAGE_VERSION}', + + description='A General Deep Learning System', + + url='https://github.com/apache/incubator-singa', + + author='NUS Database Group', + author_email='[email protected]', + + license='Apache 2', + + classifiers=[ + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + 'Intended Audience :: Developers', + 'Topic :: Deep Learning System ', + + 'License :: Apache License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], + + keywords='deep learning singa apache', + + packages= ['singa','singa.proto'], + + # py_modules=["singa"], + + # install_requires=['peppercorn'], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + #extras_require={ + # 'dev': ['check-manifest'], + # 'test': ['coverage'], + #}, + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + package_data={ + 'singa': ['_singa.so'], + }, + + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa + # In this case, 'data_file' will be installed into '<sys.prefix>/my_data' + #data_files=[('my_data', ['data/data_file'])], + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + entry_points={ + 'console_scripts': [ + 'singa=singa:main', + ], + }, +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/tensor.py ---------------------------------------------------------------------- diff --git a/src/python/tensor.py b/src/python/tensor.py index fc95d37..3b9fa52 100644 --- a/src/python/tensor.py +++ b/src/python/tensor.py @@ -29,15 +29,9 @@ import sys import os import numpy as np -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/lib')) -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/python')) -import singa - -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/src')) -from core_pb2 import * +from . import singa + +from .proto.core_pb2 import * class Tensor(object): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/test/python/example_test_tensor.py ---------------------------------------------------------------------- diff --git a/test/python/example_test_tensor.py b/test/python/example_test_tensor.py index 9b3d6f4..c5ce5f1 100644 --- a/test/python/example_test_tensor.py +++ b/test/python/example_test_tensor.py @@ -2,12 +2,12 @@ import sys, os import numpy as np sys.path.append(os.path.join(os.path.dirname(__file__), - '../../src/python')) -from tensor import * - + '../../build/python')) sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/src')) + '../../build/lib')) +sys.path.append(os.path.join(os.path.dirname(__file__),'../../build/src')) from core_pb2 import * +from singa.tensor import * #--------------------------------------------------------- @@ -37,25 +37,25 @@ print 'transpose', t.is_transpose() print 'nDim:', t.ndim() print 'size:', t.size() print 'memsize:', t.memsize() -print 'data():', t.toarray() +print 'data():', t.to_numpy() print print '----------------------------' -print 't.data\n', t.toarray() +print 't.data\n', t.to_numpy() q = t.copy() w = t.deepcopy() -print 'q.data\n', q.toarray() -print 'w.data\n', w.toarray() +print 'q.data\n', q.to_numpy() +print 'w.data\n', w.to_numpy() t += 1.23 -print 'q.data\n', q.toarray() -print 'w.data\n', w.toarray() +print 'q.data\n', q.to_numpy() +print 'w.data\n', w.to_numpy() print '----------------------------' shape = (2, 3) t.reshape(shape) print 'shape = (3, 2)' print 'after reshape, t.shape():', t.shape() -print 't.data(): \n', t.toarray() +print 't.data(): \n', t.to_numpy() shape = (3, 2) t0 = reshape(t, shape) print 'shape = (2, 3)' @@ -66,110 +66,110 @@ print print '----------------------------' t += 1.2345 -print 't += 1.234, i.e., t.__iadd__(1.2345): \n', t.toarray() +print 't += 1.234, i.e., t.__iadd__(1.2345): \n', t.to_numpy() print t1 = t print 'copy\nt1 = t' print 't1.shape():', t1.shape() -print 't1.data(): \n', t1.toarray() +print 't1.data(): \n', t1.to_numpy() print r = t1.transpose() -print 'r = t1.transpose()\nr.data() \n', r.toarray() +print 'r = t1.transpose()\nr.data() \n', r.to_numpy() print '----------------------------' t2 = log(t1) -print 't2 = log(t1): \n', t2.toarray() +print 't2 = log(t1): \n', t2.to_numpy() print t1 += t2 -print 't1 += t2, i.e., t1.__iadd__(t2): \n', t1.toarray() +print 't1 += t2, i.e., t1.__iadd__(t2): \n', t1.to_numpy() print t1 *= 2 -print 't1 *= 2, i.e., t1.__imul__(2): \n', t1.toarray() +print 't1 *= 2, i.e., t1.__imul__(2): \n', t1.to_numpy() print print '----------------------------' tc = t2.clone() -print 'clone\ntc = t2.clone()\ntc.data(): \n', tc.toarray() +print 'clone\ntc = t2.clone()\ntc.data(): \n', tc.to_numpy() print print 'sum(tc) \n', sum(tc) print t3 = sum(tc,0) -print 'sum(tc,0) \n', t3.toarray() +print 'sum(tc,0) \n', t3.to_numpy() t3 = sum(tc,1) -print 'sum(tc,1) \n', t3.toarray() +print 'sum(tc,1) \n', t3.to_numpy() print t3 = average(tc,0) -print 'average(tc,0) \n', t3.toarray() +print 'average(tc,0) \n', t3.to_numpy() t3 = average(tc,1) -print 'average(tc,1) \n', t3.toarray() +print 'average(tc,1) \n', t3.to_numpy() print t3 = softmax(tc,0) -print 'softmax(tc,0)\n', t3.toarray() +print 'softmax(tc,0)\n', t3.to_numpy() t3 = softmax(tc,1) -print 'softmax(tc,1)\n', t3.toarray() +print 'softmax(tc,1)\n', t3.to_numpy() print '----------------------------' -print 't1 \n', t1.toarray() +print 't1 \n', t1.to_numpy() print n = t1 + t2 -print 't1 + t2: \n', n.toarray() +print 't1 + t2: \n', n.to_numpy() print n = t1 * t2 -print 't1*t2: \n', n.toarray() +print 't1*t2: \n', n.to_numpy() print n = t1 - 1.2 -print 't1 - 1.2 \n', n.toarray() +print 't1 - 1.2 \n', n.to_numpy() print n = add(t1, t1) -print 'add(t1, t1) \n', n.toarray() +print 'add(t1, t1) \n', n.to_numpy() print n = add(t1, 3.4) -print 'add(t1, 3.4) \n', n.toarray() +print 'add(t1, 3.4) \n', n.to_numpy() print n = div(t1, 2.0) -print 'div(t1, 2.0) \n', n.toarray() +print 'div(t1, 2.0) \n', n.to_numpy() print print '----------------------------' shape = (2, 2) t4 = Tensor(shape) t4 += 3.45 -print 't4 += 3.45 \n', t4.toarray() +print 't4 += 3.45 \n', t4.to_numpy() print n = t4 < 3.45 -print 't4 < 3.45 \n', n.toarray() +print 't4 < 3.45 \n', n.to_numpy() print n = lt(t4, 3.45) -print 'lt(t4, 3.45) \n', n.toarray() +print 'lt(t4, 3.45) \n', n.to_numpy() print n = ge(t4, 3.45) -print 'ge(t4, 3.45) \n', n.toarray() +print 'ge(t4, 3.45) \n', n.to_numpy() print print '----------------------------' -print t1.toarray() -print tc.toarray() +print t1.to_numpy() +print tc.to_numpy() print copy_data_to_from(t1, tc, 2) -print t1.toarray() -print tc.toarray() +print t1.to_numpy() +print tc.to_numpy() #ttt = t1.singa_tensor < 5.2 #ttt = lessthan(t1, 5.2) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/test/python/test_tensor.py ---------------------------------------------------------------------- diff --git a/test/python/test_tensor.py b/test/python/test_tensor.py index d15e9ed..de9012e 100644 --- a/test/python/test_tensor.py +++ b/test/python/test_tensor.py @@ -27,14 +27,12 @@ import math import unittest import numpy as np -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../src/python')) -from tensor import * -from device import * - -sys.path.append(os.path.join(os.path.dirname(__file__), - '../../build/src')) -from core_pb2 import * +sys.path.append(os.path.join(os.path.dirname(__file__),'../../build/python')) + +from singa.tensor import * +from singa.device import * + +from singa.proto.core_pb2 import * class TestTensorMethods(unittest.TestCase):
