SINGA-108 Add Python wrapper to singa Modify paths in generatepy.sh. This script is only for developers that want to add more python APIs. Delete files in src dir. Modify singa-run.sh Delete chonho's code. Add singa.py into tool/python/.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3e58760b Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3e58760b Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3e58760b Branch: refs/heads/master Commit: 3e58760b42a134d9dad3d0f931811fa384cc0567 Parents: ef5b655 Author: xiezl <[email protected]> Authored: Mon Dec 7 11:40:37 2015 +0800 Committer: xiezl <[email protected]> Committed: Tue Dec 8 14:55:57 2015 +0800 ---------------------------------------------------------------------- Makefile.am | 4 +- bin/singa-run.sh | 10 +- include/singa/driver.h | 12 +- src/driver.cc | 2 - src/driver.i | 18 -- src/generatepy.sh | 7 - src/singa.py | 15 - tool/python/README.md | 191 ----------- tool/python/__init__.py | 0 tool/python/datasets/__init__.py | 0 tool/python/datasets/cifar10.py | 31 -- tool/python/datasets/mnist.py | 29 -- tool/python/datasets/rnnlm.py | 23 -- tool/python/ex_cifar10_cnn.py | 36 --- tool/python/ex_mnist_mlp.py | 29 -- tool/python/ex_rnnlm.py | 23 -- tool/python/model.py | 580 ---------------------------------- tool/python/singa.py | 20 ++ tool/python/singa/generatepy.sh | 11 +- tool/python/utils/__init__.py | 0 tool/python/utils/message.py | 50 --- tool/python/utils/utility.py | 45 --- 22 files changed, 42 insertions(+), 1094 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/Makefile.am ---------------------------------------------------------------------- diff --git a/Makefile.am b/Makefile.am index a09a12f..25f481c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -228,7 +228,7 @@ clean-local: rm -rf $(PROTO_SRCS) $(PROTO_HDRS) rm -rf $(PROTO_PYS) rm -rf rat_check - rm -rf tool/pb2 + rm -rf tool/python/pb2 rat: @if test ! -z '$(shell command -v java 2>/dev/null)'; then \ @@ -246,9 +246,11 @@ rat: $(NVCC) $(MSHADOW_FLAGS) $(CUDA_CFLAGS) $(CUDA_LDFLAGS) $(CUDA_LIBS) -I$(top_srcdir)/include -std=c++11 -G -c -o $@ $< install-pyLTLIBRARIES: $(py_LTLIBRARIES) + touch tool/python/singa/__init__.py cp -f .libs/_driver.so tool/python/singa/ uninstall-pyLTLIBRARIES: + rm -f tool/python/singa/__init__.py rm -f tool/python/singa/_driver.so test: singatest http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/bin/singa-run.sh ---------------------------------------------------------------------- diff --git a/bin/singa-run.sh b/bin/singa-run.sh index 1dce339..548f8f1 100755 --- a/bin/singa-run.sh +++ b/bin/singa-run.sh @@ -33,8 +33,7 @@ usage="Usage: singa-run.sh -conf <job config file> [ other arguments ]\n # parse arguments # make sure we have '-conf' and remove '-exec' -#exe=./singa -exe=./src/singa.py +exe=./singa while [ $# != 0 ]; do if [ $1 == "-exec" ]; then shift @@ -82,12 +81,9 @@ host_file=$log_dir/job.hosts ./singatool genhost $job_conf 1>$host_file || exit 1 # set command to run singa -#singa_run="$exe $args -conf $job_conf \ -# -singa_conf $SINGA_HOME/conf/singa.conf \ -# -singa_job $job_id" -singa_run="python $exe $args -conf $job_conf \ +singa_run="$exe $args -conf $job_conf \ -singa_conf $SINGA_HOME/conf/singa.conf \ - -singa_job $job_id" + -singa_job $job_id" singa_sshrun="cd $SINGA_HOME; $singa_run" # ssh and start singa processes http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/include/singa/driver.h ---------------------------------------------------------------------- diff --git a/include/singa/driver.h b/include/singa/driver.h index c7d2606..1c73db1 100644 --- a/include/singa/driver.h +++ b/include/singa/driver.h @@ -56,9 +56,19 @@ class Driver { * files. * @param[in] job_conf job configuration. */ - void Train(bool resume, const std::string str); void Train(bool resume, const JobProto& job_conf); /** + * Used for python binding. Users can also directly call it as a C++ API. + * + * It completes the functions as defined above but accept serialized string + * parameters. + * + * @param[in] resume if true resume the training from the latest checkpoint + * files. + * @param[in] str serialized string recorded job configuration. + */ + void Train(bool resume, const std::string str); + /** * Create workers and servers to conduct the training. * * @param[in] job_conf job configuration with all necessary fields set (e.g., http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/driver.cc ---------------------------------------------------------------------- diff --git a/src/driver.cc b/src/driver.cc index 1181b99..816e0df 100644 --- a/src/driver.cc +++ b/src/driver.cc @@ -153,8 +153,6 @@ void Driver::Init(int argc, char **argv) { void Driver::Train(bool resume, const std::string str){ JobProto job_conf; job_conf.ParseFromString(str); - printf("STR IS !@#!#!@# %s\n",str.c_str()); - printf("!!!!!!!!!!!!!!!!%s\n",job_conf.cluster().workspace().c_str()); Train(resume,job_conf); } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/driver.i ---------------------------------------------------------------------- diff --git a/src/driver.i b/src/driver.i deleted file mode 100644 index d1154a8..0000000 --- a/src/driver.i +++ /dev/null @@ -1,18 +0,0 @@ -%module driver -%include "std_vector.i" -%include "std_string.i" -%include "argcargv.i" -%apply (int ARGC, char **ARGV) { (int argc, char **argv) } -%{ -#include "../include/singa/driver.h" -%} - -namespace singa{ -using std::vector; -class Driver{ -public: -void Train(bool resume, const std::string job_conf); -void Init(int argc, char **argv); -}; -} - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/generatepy.sh ---------------------------------------------------------------------- diff --git a/src/generatepy.sh b/src/generatepy.sh deleted file mode 100644 index e1bbeee..0000000 --- a/src/generatepy.sh +++ /dev/null @@ -1,7 +0,0 @@ - -swig -c++ -python driver.i -g++ -fPIC driver.cc driver_wrap.cxx -shared -o _driver.so --L/home/zhongle/incubator-singa/.libs/ -lsinga -DMSHADOW_USE_CUDA=0 --DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -DUSE_GPU -std=c++11 --I/usr/cuda-7.5/include -I/home/zhongle/local/include -I../include --I/usr/include/python2.7/ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/singa.py ---------------------------------------------------------------------- diff --git a/src/singa.py b/src/singa.py deleted file mode 100644 index 2201f67..0000000 --- a/src/singa.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import sys -import string -import driver -from optparse import OptionParser - -if __name__ == '__main__': - d = driver.Driver(); - d.Init(sys.argv); - i = sys.argv.index("-conf") - s = open(sys.argv[i+1], 'r').read() - s = str(s) - print s - d.Train(False, s) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/README.md ---------------------------------------------------------------------- diff --git a/tool/python/README.md b/tool/python/README.md deleted file mode 100644 index f1b3f4d..0000000 --- a/tool/python/README.md +++ /dev/null @@ -1,191 +0,0 @@ - SINGAROOT/tool - |-- pb2 (has job_pb2.py) - |-- python - |-- model.py - |-- ex_cifar10_cnn.py - |-- ex_mnist_mlp.py - |-- datasets - |-- cifar10.py - |-- mnist.py - |-- utils - |-- utility.py - |-- message.py - -### Layer class (inherited) - -* Data -* Dense -* Activation -* Convolution2D -* MaxPooling2D -* AvgPooling2D -* LRN2D -* Dropout - -### Other classes - -* Store -* Parameter -* SGD -* Cluster - -### Model class - -* Model class has `jobconf` (JobProto) and `layers` (layer list) - -Methods in Model class - -* add - * add Layer into Model - -* compile - * set Updater (i.e., optimizer) and Cluster (i.e., topology) components - -* fit - * set Training data and parameter values for the training - * (optional) set Validatiaon data and parameter values - * set Train_one_batch component - -* evaluate - * set Testing data and parameter values for the testing - * run singa (train/test/validation) via a command script - * recieve train/test/validation results, e.g., accuracy - * [IN PROGRESS] run singa via a wrapper for Driver class - -* [IN PROGRESS] run singa with checkpoint -* [IN PROGRESS] run singa for particular tasks, e.g., classification/prediction - - -## MLP Example - -An example (to generate job.conf for mnist) - -``` -X_train, X_test, workspace = mnist.load_data() - -m = Sequential('mlp') # inherited from Model - -par = Parameter(init='uniform', low=-0.05, high=0.05) -m.add(Dense(2500, w_param=par, b_param=par)) -m.add(Activation('tanh')) -m.add(Dense(2000, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(1500, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(1000, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(500, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) - -sgd = SGD(lr=0.001, lr_type='step') -topo = Cluster(workspace) -m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) -m.fit(X_train, train_steps=1000, disp_freq=10) -result = m.evaluate(X_test, batch_size=100, test_steps=10, test_freq=60) -``` - - -## CNN Example - -An example (to generate job.conf for cifar10) - -``` -X_train, X_test, workspace = cifar10.load_data() - -m = Sequential('cnn') - -parw = Parameter(init='gauss', std=0.0001) -parb = Parameter(init='const', value=0) -m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2)) -m.add(MaxPooling2D(pool_size(3,3), stride=2)) -m.add(Activation('relu')) -m.add(LRN2D(3, alpha=0.00005, beta=0.75)) - -m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb)) -m.add(Activation('relu')) -m.add(AvgPooling2D(pool_size(3,3), stride=2)) -m.add(LRN2D(3, alpha=0.00005, beta=0.75)) - -m.add(Convolution(64, 5, 1, 2, w_param=parw, b_param=parb.setval(lr_scale=1))) -m.add(Activation('relu')) -m.add(AvgPooling2D(pool_size(3,3), stride=2)) - -parw.setval(wd_scale=250) -parb.setval(lr_scale=2, wd_scale=0) -m.add(Dense(10, w_param=parw, b_param=parb, activation='softmax')) - -sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) -topo = Cluster(workspace) -m.compile(updater=sgd, cluster=topo) -m.fit(X_train, 1000, disp_freq=30) -result = m.evaluate(X_test, 1000, test_steps=30, test_freq=300) -``` - -### TIPS - -Hidden layers for MLP can be written as -``` -par = Param(init='uniform', low=-0.05, high=0.05) -for n in [2500, 2000, 1500, 1000, 500]: - m.add(Dense(n, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) -``` - -Alternative ways to write the hidden layers -``` -m.add(Dense(2500, w_param=par, b_param=par)) -m.add(Activation('tanh')) -``` -``` -m.add(Dense(2500, init='uniform', activation='softmax')) -``` -``` -m.add(Dense(2500, w_param=Param(init='uniform'), b_param=Param(init='gaussian'))) -``` - -Alternative ways to add Data layer -``` -X_train, X_test = mnist.load_data() // parameter values are set in load_data() -m.fit(X_train, ...) // Data layer for training is added -m.evaluate(X_test, ...) // Data layer for testing is added -``` -``` -X_train, X_test = mnist.load_data() // parameter values are set in load_data() -m.add(X_train) // explicitly add Data layer -m.add(X_test) // explicitly add Data layer -``` -``` -store = Store(path='train.bin', batch_size=64, ...) // parameter values are set explicitly -m.add(Data(load='recordinput', phase='train', conf=store)) // Data layer is added -store = Store(path='test.bin', batch_size=100, ...) // parameter values are set explicitly -m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added -``` - -### Parameter class - -Users need to set parameter configuration and initial values. For example, - -* Parameter configuration - * lr = (float) // learning rate - * wd = (float) // weight decay - -* Parameter initialization - * init = (string) // one of the types, 'uniform', 'constant', 'gaussian' - * for uniform [default] - * high = (float) - * low = (float) - * for constant - * value = (float) - * for gaussian - * mean = (float) - * std = (float) - -Several ways to set Parameter values -``` -parw = Parameter(lr=2, wd=10, init='constant', value=0) -m.add(Dense(10, w_param=parw, ...) -``` -``` -parw = Parameter(init='constant', value=0) -m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...) -``` - - - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/__init__.py b/tool/python/__init__.py deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/__init__.py b/tool/python/datasets/__init__.py deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/cifar10.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/cifar10.py b/tool/python/datasets/cifar10.py deleted file mode 100644 index c959186..0000000 --- a/tool/python/datasets/cifar10.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -from model import * - -def load_data( - workspace = 'examples/cifar10', - path_mean = 'examples/cifar10/image_mean.bin', - backend = 'kvfile', - batchsize = 64, - random = 5000, - shape = (3, 32, 32), - std = 127.5, - mean = 127.5 - ): - - path_train = workspace + '/train_data.bin' - path_test = workspace + '/test_data.bin' - - store = Store(path=path_train, mean_file=path_mean, backend=backend, - random_skip=random, batchsize=batchsize, - shape=shape) - - data_train = Data(load='recordinput', phase='train', conf=store) - - store = Store(path=path_test, mean_file=path_mean, backend=backend, - batchsize=batchsize, - shape=shape) - - data_test = Data(load='recordinput', phase='test', conf=store) - - return data_train, data_test, workspace - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/mnist.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/mnist.py b/tool/python/datasets/mnist.py deleted file mode 100644 index ee04ae9..0000000 --- a/tool/python/datasets/mnist.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -from model import * - -def load_data( - workspace = 'examples/mnist', - backend = 'kvfile', - random = 5000, - batchsize = 64, - shape = 784, - std = 127.5, - mean = 127.5 - ): - - path_train = workspace + '/train_data.bin' - path_test = workspace + '/test_data.bin' - - store = Store(path=path_train, backend=backend, - random_skip=random, - batchsize=batchsize, shape=shape, - std_value=std, mean_value=mean) - data_train = Data(load='recordinput', phase='train', conf=store) - - store = Store(path=path_test, backend=backend, - batchsize=batchsize, shape=shape, - std_value=std, mean_value=mean) - data_test = Data(load='recordinput', phase='test', conf=store) - - return data_train, data_test, workspace - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/rnnlm.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/rnnlm.py b/tool/python/datasets/rnnlm.py deleted file mode 100644 index 7a2a3d5..0000000 --- a/tool/python/datasets/rnnlm.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -from model import * - -def load_data( - workspace = 'examples/rnnlm', - backend = 'kvfile', - max_window = 10 - ): - - path_train = workspace + '/train_data.bin' - path_valid = workspace + '/valid_data.bin' - path_test = workspace + '/test_data.bin' - - - data_train = Data(load='kData', phase='train', path=path_train, backend=backend, max_window=max_window) - - data_valid = Data(load='kData', phase='val', path=path_valid, max_window=max_window) - - #store = Store(path=path_test, backend=backen) - #data_test = Data(load='recordinput', phase='test', conf=store) - - return data_train, data_valid, workspace - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_cifar10_cnn.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_cifar10_cnn.py b/tool/python/ex_cifar10_cnn.py deleted file mode 100644 index fa96b48..0000000 --- a/tool/python/ex_cifar10_cnn.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -from model import * -from datasets import cifar10 - -X_train, X_test, workspace = cifar10.load_data() - -m = Sequential('cnn', label=False) - -parw = Parameter(init='gaussian', std=0.0001) -parb = Parameter(init='constant', value=0) -m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2)) -m.add(MaxPooling2D(pool_size=(3,3), stride=2)) -m.add(Activation('relu')) -m.add(LRN2D(3, alpha=0.00005, beta=0.75)) - -parw.update(std=0.01) -m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb)) -m.add(Activation('relu')) -m.add(AvgPooling2D(pool_size=(3,3), stride=2)) -m.add(LRN2D(3, alpha=0.00005, beta=0.75)) - -m.add(Convolution2D(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1)) -m.add(Activation('relu')) -m.add(AvgPooling2D(pool_size=(3,3), stride=2)) - -m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax')) - -sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) -topo = Cluster(workspace) -m.compile(optimizer=sgd, cluster=topo) -m.fit(X_train, train_steps=1000, disp_freq=30) -result = m.evaluate(X_test, test_steps=10, test_freq=300) - -#print -#m.display() - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_mnist_mlp.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_mnist_mlp.py b/tool/python/ex_mnist_mlp.py deleted file mode 100644 index 2d135d8..0000000 --- a/tool/python/ex_mnist_mlp.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -from model import * -from datasets import mnist - -X_train, X_test, workspace = mnist.load_data() - -m = Sequential('mlp') - -par = Parameter(init='uniform', range=0.05) -m.add(Dense(2500, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(2000, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(1500, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(1000, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(500, w_param=par, b_param=par, activation='tanh')) -m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) - -sgd = SGD(lr=0.001, lr_type='step') -topo = Cluster(workspace) -m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) -m.fit(X_train, train_steps=100, disp_freq=10) -result = m.evaluate(X_test, batch_size=100, test_steps=10) - -#TODO---- classify/predict for new data -#label = m.predict(data_new, ...) -#------- - -#print -#m.display() - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_rnnlm.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_rnnlm.py b/tool/python/ex_rnnlm.py deleted file mode 100644 index 69cc7b0..0000000 --- a/tool/python/ex_rnnlm.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -from model import * -from datasets import rnnlm - -vocab_size = 3720 - -X_train, X_valid, workspace = rnnlm.load_data() - -m = Sequential('rnnlm', label=False) - -parw = Parameter(init='uniform', range=0.3) -m.add(Embedding(in_dim=vocab_size, out_dim=15, w_param=parw)) -m.add(RNNLM(1, w_param=parw)) - -sgd = SGD(lr_type='fixed', step=(0,48810,56945,65080,73215), step_lr=(0.1,0.05,0.025,0.0125,0.00625)) -topo = Cluster(workspace) -m.compile(loss='user_loss_rnnlm', in_dim=vocab_size, nclass=100, optimizer=sgd, cluster=topo) - -m.fit(X_train, train_steps=81350, disp_freq=8135) -result = m.evaluate(X_valid, validate_steps=683, validate_freq=8135, execpath='examples/rnnlm/rnnlm.bin') - -#print -#m.display() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/model.py ---------------------------------------------------------------------- diff --git a/tool/python/model.py b/tool/python/model.py deleted file mode 100644 index a558a18..0000000 --- a/tool/python/model.py +++ /dev/null @@ -1,580 +0,0 @@ -#!/usr/bin/env python -import os, sys, re, subprocess -from utils.utility import * -from utils.message import * -from google.protobuf import text_format -#sys.path.append(os.path.join(os.path.dirname(__file__), '../pb2')) -#from job_pb2 import * -#from rnnlm_pb2 import * - -class Model(object): - - def __init__(self, name='my model', label=False): - self.jobconf = Message('Job', name=name).proto - self.layers = [] - self.label = label - - def add(self, layer): - self.layers.append(layer) - - def compile(self, optimizer=None, cluster=None, loss='categorical_crossentropy', topk=1, **kwargs): - ''' - required - optimizer = (Updater) // updater settings, e.g., SGD - cluster = (Cluster) // cluster settings - optional - loss = (string) // name of loss function type - topk = (int) // the number of results considered to compute accuracy - ''' - assert optimizer != None, 'optimizer (Updater) should be set' - assert cluster != None, 'cluster (Cluster) should be set' - setval(self.jobconf, updater=optimizer.proto) - setval(self.jobconf, cluster=cluster.proto) - - # take care of loss function layer - assert loss != None, 'loss should be set' - if hasattr(self.layers[len(self.layers)-1], 'mask'): - ly = self.layers[len(self.layers)-1].mask - else: - ly = self.layers[len(self.layers)-1].layer - - if ly.type == enumLayerType('softmax'): - # revise the last layer - if loss == 'categorical_crossentropy': - setval(ly, type=enumLayerType('softmaxloss')) - setval(ly.softmaxloss_conf, topk=topk) - elif loss == 'mean_squared_error': - setval(ly, type=enumLayerType('euclideanloss')) - else: - # add new layer - if loss == 'categorical_crossentropy': - self.add(Activation('softmaxloss', topk=topk)) - elif loss == 'mean_squared_error': - self.add(Activation('euclideanloss')) - elif loss == 'user_loss_rnnlm': # user-defined loss layer for rnnlm - self.add(UserLossRNNLM(nclass=kwargs['nclass'], vocab_size=kwargs['in_dim'])) - - def build(self): - net = NetProto() - slyname = self.layers[0].layer.name - for i in range(len(self.layers)): - ly = net.layer.add() - ly.CopyFrom(self.layers[i].layer) - lastly = ly - if self.layers[i].is_datalayer == True: - continue - getattr(ly, 'srclayers').append(slyname) - slyname = ly.name - if hasattr(self.layers[i], 'mask'): - mly = net.layer.add() - mly.CopyFrom(self.layers[i].mask) - getattr(mly, 'srclayers').append(slyname) - slyname = mly.name - lastly = mly - - if self.label == True: - label_layer = Layer(name='label', type=kLabel) - ly = net.layer.add() - ly.CopyFrom(label_layer.layer) - getattr(ly, 'srclayers').append(self.layers[0].layer.name) - getattr(lastly, 'srclayers').append(label_layer.layer.name) - else: - getattr(lastly, 'srclayers').append(self.layers[0].layer.name) - - setval(self.jobconf, neuralnet=net) - - def fit(self, **kwargs): - pass - - def evaluate(self, **kwargs): - pass - - def display(self): - print text_format.MessageToString(self.jobconf) - - -class Sequential(Model): - def __init__(self, name='my model', label=False): - super(Sequential, self).__init__(name=name, label=label) - - def exist_datalayer(self, phase): - if self.layers[0].layer.include != enumPhase(phase): # train - if self.layers[1].layer.include != enumPhase(phase): # test - if self.layers[2].layer.include != enumPhase(phase): # valid - return False - return True - - def fit(self, data=None, train_steps=0, **fields): - ''' - required - data = (Data) // Data class object for training data - train_steps = (int) // the number of training, i.e., epoch - optional - **fields (KEY=VALUE) - batch_size = (int) // batch size for training data - disp_freq = (int) // frequency to display training info - disp_after = (int) // display after this number - validate_data = (Data) // validation data, specified in load_data() - validate_freq = (int) // frequency of validation - validate_steps = (int) // total number of steps for validation - validate_after = (int) // start validation after this number - checkpoint_path = (string) // path to checkpoint file - checkpoint_freq = (int) // frequency for checkpoint - checkpoint_after = (int) // start checkpointing after this number - ''' - assert data != None, 'Training data shold be set' - assert train_steps != 0, 'Training steps shold be set' - - if 'batch_size' in fields: # if new value is set, replace the batch_size - setval(data.layer.store_conf, batchsize=fields['batch_size']) - - if self.exist_datalayer('train') == False: - self.layers.insert(0, data) - - if 'validate_data' in fields: - self.layers.insert(1, fields['validate_data']) - - setval(self.jobconf, train_steps=train_steps) - setval(self.jobconf, **fields) - - # set Train_one_batch component, using backprogapation - setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType('bp')).proto) - - def evaluate(self, data=None, execpath='', **fields): - ''' - required - data = (Data) // Data class object for testing data - optional - test_steps = (int) // the number of testing - **fields (KEY=VALUE) - batch_size = (int) // batch size for testing data - test_freq = (int) // frequency of testing - test_steps = (int) // total number of steps for testing - test_after = (int) // start testing after this number of steps - ''' - assert data != None, 'Testing data shold be set' - - if 'batch_size' in fields: # if new value is set, replace the batch_size - setval(data.layer.store_conf, batchsize=fields['batch_size']) - - if self.exist_datalayer('test') == False: - self.layers.insert(1, data) - - setval(self.jobconf, **fields) - - self.build() # construct Nneuralnet Component - - with open('job.conf', 'w') as f: - f.write(text_format.MessageToString(self.jobconf)) - - #self.display() - self.result = SingaRun(execpath=execpath) - return self.result - -class Store(object): - def __init__(self, **kwargs): - ''' - **kwargs - path = (string) // path to dataset - backend = (string) // - batch_size = (int) // batch size of dataset - shape = (int) // - - ''' - self.proto = Message('Store', **kwargs).proto - -class Algorithm(object): - def __init__(self, type=enumAlgType('bp'), **kwargs): - alg = Message('Alg', alg=type, **kwargs).proto - if type == enumAlgType('cd'): - setval(alg.cd_conf, **kwargs) - self.proto = alg - -class SGD(object): - def __init__(self, lr=0.01, lr_type=None, - decay=0, momentum=0, - step=(0), step_lr=(0.01), **fields): - ''' - required - lr = (float) // base learning rate - lr_type = (string) // type of learning rate - optional - **fields (KEY=VALUE) - decay = (float) // weight decay - momentum = (float) // momentum - - ''' - assert lr - assert lr_type != None, 'type of learning rate should be specified' - - upd = Message('Updater', type=kSGD, **fields).proto - setval(upd.learning_rate, base_lr=lr) - if decay > 0: - setval(upd, weight_decay=decay) - if momentum > 0: - setval(upd, momentum=momentum) - - if lr_type == 'step': - cp = Message('Step', change_freq=60, gamma=0.997) - setval(upd.learning_rate, type=kStep, step_conf=cp.proto) - elif lr_type == 'fixed': - cp = Message('FixedStep', step=step, step_lr=step_lr) - setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto) - elif lr_type == 'linear': - cp = Message('Linear', change_freq=10, final_lr=0.1) - setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto) - self.proto = upd - - -class Cluster(object): - def __init__(self, workspace=None, - nworker_groups=1, nserver_groups=1, - nworkers_per_group=1, nservers_per_group=1, - nworkers_per_procs=1, nservers_per_procs=1, - **fields): - ''' - required - workspace = (string) // workspace path - optional - nworker_groups = (int) - nserver_groups = (int) - nworkers_per_group = (int) - nservers_per_group = (int) - nworkers_per_procs = (int) - nservers_per_procs = (int) - **fields - server_worker_separate = (bool) - ''' - assert workspace != None, 'need to set workspace' - self.proto = Message('Cluster', workspace=workspace).proto - # optional - self.proto.nworker_groups = nworker_groups - self.proto.nserver_groups = nserver_groups - self.proto.nworkers_per_group = nworkers_per_group - self.proto.nservers_per_group = nservers_per_group - self.proto.nworkers_per_procs = nworkers_per_procs - self.proto.nservers_per_procs = nservers_per_procs - # other fields - setval(self.proto, **fields) - -#TODO make this internally used -class Parameter(object): - - def __init__(self, **kwargs): - ''' - optional - **kwargs - name = (string) // parameter name - lr = (float) // learning rate - wd = (float) // weight decay - init = (string) // initialized type {'constant','uniform','gaussian'} - value = (int) // value for 'constant' - range = (float) // [low, high] for 'uniform', low=-range, high=range - low = (float) // low value for 'uniform' - high = (float) // high value for 'uniform' - mean = (float) // mean for 'gaussian' - std = (float) // std for 'gaussian' - ''' - fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1, - 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1 - } - self.param = Message('Param', **fields).proto - - if not 'name' in kwargs: - setval(self.param, name=generateName('param', 1)) - else: - setval(self.param, name=kwargs['name']) - - if 'range' in kwargs: - kwargs['low'] = -float(kwargs['range']) - kwargs['high'] = float(kwargs['range']) - - if 'init' in kwargs: - pg = Message('ParamGen', type=enumInitMethod(kwargs['init']), **kwargs) - del kwargs['init'] - else: # default: uniform - pg = Message('ParamGen', type=enumInitMethod(kwargs['uniform'])) - setval(self.param, init=pg.proto) - - - #if 'param_init' in kwargs: - # setval(self.param, init=kwargs['param_init'].proto) - # del kwargs['param_init'] - - def update(self, **fields): - setval(self.param, **fields) - setval(self.param.init, **fields) - - -class Layer(object): - def __init__(self, **kwargs): - self.layer = Message('Layer', **kwargs).proto - # required - if not 'name' in kwargs: - setval(self.layer, name=generateName('layer', 1)) - - # srclayers are set in Model.build() - self.is_datalayer = False - - def setParamField(self, param, pname, **kwargs): - # param: ParamProto - if pname == 'w': - field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else param.lr_scale, - 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else param.wd_scale - } - elif pname == 'b': - field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else param.lr_scale, - 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else param.wd_scale - } - setval(param, name=generateName(pname), **field) - -class Data(Layer): - def __init__(self, load, phase='train', conf=None, **kwargs): - assert load != None, 'data type should be specified' - if load == 'kData': - super(Data, self).__init__(name=generateName('data'), user_type=load) - else: - self.layer_type = enumLayerType(load) - super(Data, self).__init__(name=generateName('data'), type=self.layer_type) - - # include/exclude - setval(self.layer, include=enumPhase(phase)) - #setval(self.layer, exclude=kTest if phase=='train' else kTrain) - - if conf == None: - if load == 'kData': - setval(self.layer.Extensions[data_conf], **kwargs) - else: - setval(self.layer.store_conf, **kwargs) - else: - setval(self.layer, store_conf=conf.proto) - self.is_datalayer = True - -class Convolution2D(Layer): - def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0, - init='uniform', w_param=None, b_param=None, - activation=None, **kwargs): - ''' - required - nb_filter = (int) // the number of filters - kernel = (int) // the size of filter - optional - stride = (int) // the size of stride - pad = (int) // the size of padding - xxxxx - - ''' - assert nb_filter > 0 and kernel > 0, 'should be set as positive int' - super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution) - fields = {'num_filters' : nb_filter, - 'kernel' : kernel, - 'stride' : stride, - 'pad' : pad} - setval(self.layer.convolution_conf, **fields) - - # parameter w - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else w_param.param.lr_scale, - 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else w_param.param.wd_scale - } - setval(w_param.param, name=generateName('w'), **field) - setval(w_param.param.init, **kwargs) - setval(self.layer, param=w_param.param) - - # parameter b - if b_param == None: - b_param = Parameter(name=generateName('b'), init=init) # default: uniform - else: - field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else b_param.param.lr_scale, - 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else b_param.param.wd_scale - } - setval(b_param.param, name=generateName('b'), **field) - setval(self.layer, param=b_param.param) - - # following layers: e.g., activation, dropout, etc. - if activation: - self.mask = Activation(activation=activation).layer - -class MaxPooling2D(Layer): - def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): - ''' - required - pool_size = (int|tuple) // the size for pooling - optional - stride = (int) // the size of striding - ignore_border = (bool) // flag for padding - **kwargs // fields for Layer class - ''' - assert pool_size != None, 'pool_size is required' - if type(pool_size) == int: - pool_size = (pool_size, pool_size) - assert type(pool_size) == tuple and \ - pool_size[0] == pool_size[1], 'pool size should be square in Singa' - super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) - fields = {'pool' : PoolingProto().MAX, - 'kernel' : pool_size[0], - 'stride' : stride, - 'pad' : 0 if ignore_border else 1} - setval(self.layer.pooling_conf, **fields) - -class AvgPooling2D(Layer): - def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): - ''' - required - pool_size = (int|tuple) // size for pooling - optional - stride = (int) // size of striding - ignore_border = (bool) // flag for padding - **kwargs // fields for Layer class - ''' - assert pool_size != None, 'pool_size is required' - if type(pool_size) == int: - pool_size = (pool_size, pool_size) - assert type(pool_size) == tuple and \ - pool_size[0] == pool_size[1], 'pool size should be square in Singa' - super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) - self.layer.pooling_conf.pool = PoolingProto().AVG - fields = {'pool' : PoolingProto().AVG, - 'kernel' : pool_size[0], - 'stride' : stride, - 'pad' : 0 if ignore_border else 1} - setval(self.layer.pooling_conf, **fields) - -class LRN2D(Layer): - def __init__(self, size=0, alpha=1e-4, k=1, beta=0.75, **kwargs): - super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN) - # required - assert size != 0, 'local size should be set' - self.layer.lrn_conf.local_size = size - setval(self.layer.lrn_conf, alpha=alpha, knorm=k, beta=beta, **kwargs) - -class Dense(Layer): - def __init__(self, output_dim=0, activation=None, - init='uniform', w_param=None, b_param=None, input_dim=None, - **kwargs): - ''' - required - output_dim = (int) - optional - activation = (string) - **kwargs - w_lr - w_wd - - ''' - assert output_dim > 0, 'output_dim should be set' - super(Dense, self).__init__(type=kInnerProduct, **kwargs) - self.layer.innerproduct_conf.num_output = output_dim # required - - # parameter w - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - self.setParamField(w_param.param, 'w', **kwargs) - setval(self.layer, param=w_param.param) - - # parameter b - if b_param == None: - b_param = Parameter(name=generateName('b'), init=init) # default: uniform - else: - self.setParamField(b_param.param, 'b', **kwargs) - setval(self.layer, param=b_param.param) - - # following layers: e.g., activation, dropout, etc. - if activation: - self.mask = Activation(activation=activation).layer - -class Activation(Layer): - def __init__(self, activation='stanh', topk=1): - self.name = activation - #TODO better way? - if activation == 'tanh': activation = 'stanh' - self.layer_type = enumLayerType(activation) - super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type) - if activation == 'softmaxloss': - self.layer.softmaxloss_conf.topk = topk - -class Dropout(Layer): - def __init__(self, ratio=0.5): - self.name = 'dropout' - self.layer_type = kDropout - super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type) - self.layer.dropout_conf.dropout_ratio = ratio - - -class RGB(Layer): - def __init__(self, meanfile=None, **kwargs): - assert meanfile != None, 'meanfile should be specified' - self.name = 'rgb' - self.layer_type = kRGBImage - super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type) - self.layer.rgbimage_conf.meanfile = meanfile - -class Embedding(Layer): - def __init__(self, in_dim, out_dim, w_param=None, **kwargs): - super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding') - fields = { 'vocab_size': in_dim, - 'word_dim': out_dim } - setval(self.layer.Extensions[embedding_conf], **fields) - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - self.setParamField(w_param.param, 'w', **kwargs) - setval(self.layer, param=w_param.param) - -class RNNLM(Layer): - def __init__(self, dim, w_param=None, **kwargs): - super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden') - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - self.setParamField(w_param.param, 'w', **kwargs) - setval(self.layer, param=w_param.param) - -class UserLossRNNLM(Layer): - def __init__(self, **kwargs): - super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss') - self.layer.Extensions[loss_conf].nclass = kwargs['nclass'] - self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size'] - setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', range=0.3).param) - setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', range=0.3).param) - - -#TODO run singa training/testing via a wrapper for Driver -def SingaRun(execpath=''): - SINGAROOT = '../../' - conf = 'job.conf' - if execpath=='': - cmd = '../../bin/singa-run.sh ' \ - + '-conf %s ' % conf - else: - cmd = '../../bin/singa-run.sh ' \ - + '-conf %s ' % conf \ - + '-exec %s ' % execpath - - procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - - resultDic = {} - output = iter(procs.stdout.readline, '') - for line in output: - print line[:-1] - line = re.findall(r'[\w|*.*]+', line) - if 'accuracy' in line: - step = line[line.index('step')+1] - acc = line[line.index('accuracy')+1] - loss = line[line.index('loss')+1] - #print 'Step: ', line[idx_step+1], 'Acc: ', acc, 'Loss: ', loss - resultDic.setdefault(step,{})['acc'] = acc - resultDic.setdefault(step,{})['loss'] = loss - elif 'Train' in line: - step = line[line.index('step')+1] - loss = line[line.index('loss')+1] - ppl = line[line.index('ppl')+1] - resultDic.setdefault(step,{})['loss'] = loss - resultDic.setdefault(step,{})['ppl'] = ppl - - #TODO better format to store?? - return resultDic - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/singa.py ---------------------------------------------------------------------- diff --git a/tool/python/singa.py b/tool/python/singa.py new file mode 100755 index 0000000..4c41729 --- /dev/null +++ b/tool/python/singa.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + + +import os +import sys +import string +import pb2.job_pb2 as job_pb2 +import singa.driver as driver +from google.protobuf.text_format import Merge + +if __name__ == '__main__': + i = sys.argv.index("-conf") + s = open(sys.argv[i+1], 'r').read() + s = str(s) + j = job_pb2.JobProto() + Merge(s,j) + b = j.SerializeToString() + d = driver.Driver() + d.Init(sys.argv) + d.Train(False,b) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/singa/generatepy.sh ---------------------------------------------------------------------- diff --git a/tool/python/singa/generatepy.sh b/tool/python/singa/generatepy.sh index bb4a3af..2a9b2e4 100755 --- a/tool/python/singa/generatepy.sh +++ b/tool/python/singa/generatepy.sh @@ -1,8 +1,7 @@ - +#!/usr/bin/env bash swig -c++ -python driver.i -g++ -fPIC /home/zhongle/incubator-singa/src/driver.cc driver_wrap.cxx -shared -o _driver.so \ - -L/home/zhongle/incubator-singa/.libs/ -lsinga -DMSHADOW_USE_CUDA=0 \ - -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -DUSE_GPU -std=c++11 \ - -I/usr/cuda-7.5/include -I/home/zhongle/local/include \ - -I/home/zhongle/incubator-singa/include \ +g++ -fPIC /../../../src/driver.cc driver_wrap.cxx -shared -o _driver.so \ + -L../../../.libs/ -lsinga -DMSHADOW_USE_CUDA=0 \ + -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -std=c++11 \ + -I/../../../include \ -I/usr/include/python2.7/ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/utils/__init__.py b/tool/python/utils/__init__.py deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/message.py ---------------------------------------------------------------------- diff --git a/tool/python/utils/message.py b/tool/python/utils/message.py deleted file mode 100644 index 10a1a18..0000000 --- a/tool/python/utils/message.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -import sys, os -from utility import * -sys.path.append(os.path.join(os.path.dirname(__file__),'../../pb2')) - -module_list=[] - -# import all modules in dir singa_root/too/pb2, except common, singa and __init__ -for f in os.listdir(os.path.join(os.path.dirname(__file__),'../../pb2')): - if (f.endswith(".pyc")): - continue - if(f == "__init__.py" or f == "common_pb2.py" or f == "singa_pb2.py" ): - continue - module_name = f.split('.')[0] - module=__import__(module_name) - module_list.append(module) - for func_name in dir(module): - if not func_name.startswith("__"): - globals()[func_name] = getattr(module,func_name) - -class Message(object): - def __init__(self,protoname,**kwargs): - for module in module_list: - if hasattr(module,protoname+"Proto"): - class_ = getattr(module,protoname+"Proto") - self.proto = class_() - return setval(self.proto,**kwargs) - raise Exception('invalid protoname') - -enumDict_=dict() - -#get all enum type list in modules -for module in module_list: - for enumtype in module.DESCRIPTOR.enum_types_by_name: - tempDict=enumDict_[enumtype]=dict() - for name in getattr(module,enumtype).DESCRIPTOR.values_by_name: - tempDict[name[1:].lower()]=getattr(module,name) - -def make_function(enumtype): - def _function(key): - return enumDict_[enumtype][key] - return _function - -current_module = sys.modules[__name__] - -#def all the enumtypes -for module in module_list: - for enumtype in module.DESCRIPTOR.enum_types_by_name: - setattr(current_module,"enum"+enumtype,make_function(enumtype)) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/utility.py ---------------------------------------------------------------------- diff --git a/tool/python/utils/utility.py b/tool/python/utils/utility.py deleted file mode 100644 index 42840c9..0000000 --- a/tool/python/utils/utility.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python - -layerid = 0 -paramid = 0 - -def generateName(label, op=0): - global layerid, paramid - num = layerid - if label == 'layer': - if op ==1: layerid += 1 - num = layerid - elif label == 'param': - if op ==1: paramid += 1 - num = paramid - else: - if op ==1: layerid += 1 - num = layerid - - return '{0}{1}'.format(label, num) - - -def setval(proto, **kwargs): - for k,v in kwargs.items(): - #print 'kv: ', k, ', ', v - if hasattr(proto, k): - flabel = proto.DESCRIPTOR.fields_by_name[k].label - ftype = proto.DESCRIPTOR.fields_by_name[k].type - - fattr = getattr(proto, k) - if flabel == 3: # repeated field - if ftype == 11: # message type - fattr = fattr.add() - fattr.MergeFrom(v) - else: - if type(v) == list or type(v) == tuple: - for i in range(len(v)): - fattr.append(v[i]) - else: - fattr.append(v) - else: - if ftype == 11: # message type - fattr = getattr(proto,k) - fattr.MergeFrom(v) - else: - setattr(proto, k, v)
