SINGA-108 Add Python wrapper to singa Reorganize directories. Move python related files into tool/python/.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/b8104eff Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/b8104eff Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/b8104eff Branch: refs/heads/master Commit: b8104eff724a383218be580dc743a2d86d08a6f7 Parents: 0578d82 Author: xiezl <[email protected]> Authored: Fri Dec 4 18:11:23 2015 +0800 Committer: xiezl <[email protected]> Committed: Tue Dec 8 12:45:16 2015 +0800 ---------------------------------------------------------------------- src/singa.py | 15 + tool/python/README.md | 191 ++ tool/python/__init__.py | 0 tool/python/datasets/__init__.py | 0 tool/python/datasets/cifar10.py | 31 + tool/python/datasets/mnist.py | 29 + tool/python/datasets/rnnlm.py | 23 + tool/python/ex_cifar10_cnn.py | 36 + tool/python/ex_mnist_mlp.py | 29 + tool/python/ex_rnnlm.py | 23 + tool/python/model.py | 580 ++++ tool/python/singa/__init__.py | 0 tool/python/singa/driver.i | 18 + tool/python/singa/driver.py | 120 + tool/python/singa/driver_wrap.cxx | 5334 ++++++++++++++++++++++++++++++++ tool/python/singa/generatepy.sh | 8 + tool/python/utils/__init__.py | 0 tool/python/utils/message.py | 50 + tool/python/utils/utility.py | 45 + 19 files changed, 6532 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/src/singa.py ---------------------------------------------------------------------- diff --git a/src/singa.py b/src/singa.py new file mode 100644 index 0000000..2201f67 --- /dev/null +++ b/src/singa.py @@ -0,0 +1,15 @@ +import os +import sys +import string +import driver +from optparse import OptionParser + +if __name__ == '__main__': + d = driver.Driver(); + d.Init(sys.argv); + i = sys.argv.index("-conf") + s = open(sys.argv[i+1], 'r').read() + s = str(s) + print s + d.Train(False, s) + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/README.md ---------------------------------------------------------------------- diff --git a/tool/python/README.md b/tool/python/README.md new file mode 100644 index 0000000..f1b3f4d --- /dev/null +++ b/tool/python/README.md @@ -0,0 +1,191 @@ + SINGAROOT/tool + |-- pb2 (has job_pb2.py) + |-- python + |-- model.py + |-- ex_cifar10_cnn.py + |-- ex_mnist_mlp.py + |-- datasets + |-- cifar10.py + |-- mnist.py + |-- utils + |-- utility.py + |-- message.py + +### Layer class (inherited) + +* Data +* Dense +* Activation +* Convolution2D +* MaxPooling2D +* AvgPooling2D +* LRN2D +* Dropout + +### Other classes + +* Store +* Parameter +* SGD +* Cluster + +### Model class + +* Model class has `jobconf` (JobProto) and `layers` (layer list) + +Methods in Model class + +* add + * add Layer into Model + +* compile + * set Updater (i.e., optimizer) and Cluster (i.e., topology) components + +* fit + * set Training data and parameter values for the training + * (optional) set Validatiaon data and parameter values + * set Train_one_batch component + +* evaluate + * set Testing data and parameter values for the testing + * run singa (train/test/validation) via a command script + * recieve train/test/validation results, e.g., accuracy + * [IN PROGRESS] run singa via a wrapper for Driver class + +* [IN PROGRESS] run singa with checkpoint +* [IN PROGRESS] run singa for particular tasks, e.g., classification/prediction + + +## MLP Example + +An example (to generate job.conf for mnist) + +``` +X_train, X_test, workspace = mnist.load_data() + +m = Sequential('mlp') # inherited from Model + +par = Parameter(init='uniform', low=-0.05, high=0.05) +m.add(Dense(2500, w_param=par, b_param=par)) +m.add(Activation('tanh')) +m.add(Dense(2000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) + +sgd = SGD(lr=0.001, lr_type='step') +topo = Cluster(workspace) +m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) +m.fit(X_train, train_steps=1000, disp_freq=10) +result = m.evaluate(X_test, batch_size=100, test_steps=10, test_freq=60) +``` + + +## CNN Example + +An example (to generate job.conf for cifar10) + +``` +X_train, X_test, workspace = cifar10.load_data() + +m = Sequential('cnn') + +parw = Parameter(init='gauss', std=0.0001) +parb = Parameter(init='const', value=0) +m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2)) +m.add(MaxPooling2D(pool_size(3,3), stride=2)) +m.add(Activation('relu')) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size(3,3), stride=2)) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution(64, 5, 1, 2, w_param=parw, b_param=parb.setval(lr_scale=1))) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size(3,3), stride=2)) + +parw.setval(wd_scale=250) +parb.setval(lr_scale=2, wd_scale=0) +m.add(Dense(10, w_param=parw, b_param=parb, activation='softmax')) + +sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) +topo = Cluster(workspace) +m.compile(updater=sgd, cluster=topo) +m.fit(X_train, 1000, disp_freq=30) +result = m.evaluate(X_test, 1000, test_steps=30, test_freq=300) +``` + +### TIPS + +Hidden layers for MLP can be written as +``` +par = Param(init='uniform', low=-0.05, high=0.05) +for n in [2500, 2000, 1500, 1000, 500]: + m.add(Dense(n, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) +``` + +Alternative ways to write the hidden layers +``` +m.add(Dense(2500, w_param=par, b_param=par)) +m.add(Activation('tanh')) +``` +``` +m.add(Dense(2500, init='uniform', activation='softmax')) +``` +``` +m.add(Dense(2500, w_param=Param(init='uniform'), b_param=Param(init='gaussian'))) +``` + +Alternative ways to add Data layer +``` +X_train, X_test = mnist.load_data() // parameter values are set in load_data() +m.fit(X_train, ...) // Data layer for training is added +m.evaluate(X_test, ...) // Data layer for testing is added +``` +``` +X_train, X_test = mnist.load_data() // parameter values are set in load_data() +m.add(X_train) // explicitly add Data layer +m.add(X_test) // explicitly add Data layer +``` +``` +store = Store(path='train.bin', batch_size=64, ...) // parameter values are set explicitly +m.add(Data(load='recordinput', phase='train', conf=store)) // Data layer is added +store = Store(path='test.bin', batch_size=100, ...) // parameter values are set explicitly +m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added +``` + +### Parameter class + +Users need to set parameter configuration and initial values. For example, + +* Parameter configuration + * lr = (float) // learning rate + * wd = (float) // weight decay + +* Parameter initialization + * init = (string) // one of the types, 'uniform', 'constant', 'gaussian' + * for uniform [default] + * high = (float) + * low = (float) + * for constant + * value = (float) + * for gaussian + * mean = (float) + * std = (float) + +Several ways to set Parameter values +``` +parw = Parameter(lr=2, wd=10, init='constant', value=0) +m.add(Dense(10, w_param=parw, ...) +``` +``` +parw = Parameter(init='constant', value=0) +m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...) +``` + + + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/__init__.py b/tool/python/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/datasets/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/__init__.py b/tool/python/datasets/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/datasets/cifar10.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/cifar10.py b/tool/python/datasets/cifar10.py new file mode 100644 index 0000000..c959186 --- /dev/null +++ b/tool/python/datasets/cifar10.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +from model import * + +def load_data( + workspace = 'examples/cifar10', + path_mean = 'examples/cifar10/image_mean.bin', + backend = 'kvfile', + batchsize = 64, + random = 5000, + shape = (3, 32, 32), + std = 127.5, + mean = 127.5 + ): + + path_train = workspace + '/train_data.bin' + path_test = workspace + '/test_data.bin' + + store = Store(path=path_train, mean_file=path_mean, backend=backend, + random_skip=random, batchsize=batchsize, + shape=shape) + + data_train = Data(load='recordinput', phase='train', conf=store) + + store = Store(path=path_test, mean_file=path_mean, backend=backend, + batchsize=batchsize, + shape=shape) + + data_test = Data(load='recordinput', phase='test', conf=store) + + return data_train, data_test, workspace + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/datasets/mnist.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/mnist.py b/tool/python/datasets/mnist.py new file mode 100644 index 0000000..ee04ae9 --- /dev/null +++ b/tool/python/datasets/mnist.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +from model import * + +def load_data( + workspace = 'examples/mnist', + backend = 'kvfile', + random = 5000, + batchsize = 64, + shape = 784, + std = 127.5, + mean = 127.5 + ): + + path_train = workspace + '/train_data.bin' + path_test = workspace + '/test_data.bin' + + store = Store(path=path_train, backend=backend, + random_skip=random, + batchsize=batchsize, shape=shape, + std_value=std, mean_value=mean) + data_train = Data(load='recordinput', phase='train', conf=store) + + store = Store(path=path_test, backend=backend, + batchsize=batchsize, shape=shape, + std_value=std, mean_value=mean) + data_test = Data(load='recordinput', phase='test', conf=store) + + return data_train, data_test, workspace + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/datasets/rnnlm.py ---------------------------------------------------------------------- diff --git a/tool/python/datasets/rnnlm.py b/tool/python/datasets/rnnlm.py new file mode 100644 index 0000000..7a2a3d5 --- /dev/null +++ b/tool/python/datasets/rnnlm.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from model import * + +def load_data( + workspace = 'examples/rnnlm', + backend = 'kvfile', + max_window = 10 + ): + + path_train = workspace + '/train_data.bin' + path_valid = workspace + '/valid_data.bin' + path_test = workspace + '/test_data.bin' + + + data_train = Data(load='kData', phase='train', path=path_train, backend=backend, max_window=max_window) + + data_valid = Data(load='kData', phase='val', path=path_valid, max_window=max_window) + + #store = Store(path=path_test, backend=backen) + #data_test = Data(load='recordinput', phase='test', conf=store) + + return data_train, data_valid, workspace + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/ex_cifar10_cnn.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_cifar10_cnn.py b/tool/python/ex_cifar10_cnn.py new file mode 100644 index 0000000..fa96b48 --- /dev/null +++ b/tool/python/ex_cifar10_cnn.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +from model import * +from datasets import cifar10 + +X_train, X_test, workspace = cifar10.load_data() + +m = Sequential('cnn', label=False) + +parw = Parameter(init='gaussian', std=0.0001) +parb = Parameter(init='constant', value=0) +m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2)) +m.add(MaxPooling2D(pool_size=(3,3), stride=2)) +m.add(Activation('relu')) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +parw.update(std=0.01) +m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution2D(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) + +m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax')) + +sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) +topo = Cluster(workspace) +m.compile(optimizer=sgd, cluster=topo) +m.fit(X_train, train_steps=1000, disp_freq=30) +result = m.evaluate(X_test, test_steps=10, test_freq=300) + +#print +#m.display() + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/ex_mnist_mlp.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_mnist_mlp.py b/tool/python/ex_mnist_mlp.py new file mode 100644 index 0000000..2d135d8 --- /dev/null +++ b/tool/python/ex_mnist_mlp.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +from model import * +from datasets import mnist + +X_train, X_test, workspace = mnist.load_data() + +m = Sequential('mlp') + +par = Parameter(init='uniform', range=0.05) +m.add(Dense(2500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(2000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) + +sgd = SGD(lr=0.001, lr_type='step') +topo = Cluster(workspace) +m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) +m.fit(X_train, train_steps=100, disp_freq=10) +result = m.evaluate(X_test, batch_size=100, test_steps=10) + +#TODO---- classify/predict for new data +#label = m.predict(data_new, ...) +#------- + +#print +#m.display() + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/ex_rnnlm.py ---------------------------------------------------------------------- diff --git a/tool/python/ex_rnnlm.py b/tool/python/ex_rnnlm.py new file mode 100644 index 0000000..69cc7b0 --- /dev/null +++ b/tool/python/ex_rnnlm.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from model import * +from datasets import rnnlm + +vocab_size = 3720 + +X_train, X_valid, workspace = rnnlm.load_data() + +m = Sequential('rnnlm', label=False) + +parw = Parameter(init='uniform', range=0.3) +m.add(Embedding(in_dim=vocab_size, out_dim=15, w_param=parw)) +m.add(RNNLM(1, w_param=parw)) + +sgd = SGD(lr_type='fixed', step=(0,48810,56945,65080,73215), step_lr=(0.1,0.05,0.025,0.0125,0.00625)) +topo = Cluster(workspace) +m.compile(loss='user_loss_rnnlm', in_dim=vocab_size, nclass=100, optimizer=sgd, cluster=topo) + +m.fit(X_train, train_steps=81350, disp_freq=8135) +result = m.evaluate(X_valid, validate_steps=683, validate_freq=8135, execpath='examples/rnnlm/rnnlm.bin') + +#print +#m.display() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/model.py ---------------------------------------------------------------------- diff --git a/tool/python/model.py b/tool/python/model.py new file mode 100644 index 0000000..a558a18 --- /dev/null +++ b/tool/python/model.py @@ -0,0 +1,580 @@ +#!/usr/bin/env python +import os, sys, re, subprocess +from utils.utility import * +from utils.message import * +from google.protobuf import text_format +#sys.path.append(os.path.join(os.path.dirname(__file__), '../pb2')) +#from job_pb2 import * +#from rnnlm_pb2 import * + +class Model(object): + + def __init__(self, name='my model', label=False): + self.jobconf = Message('Job', name=name).proto + self.layers = [] + self.label = label + + def add(self, layer): + self.layers.append(layer) + + def compile(self, optimizer=None, cluster=None, loss='categorical_crossentropy', topk=1, **kwargs): + ''' + required + optimizer = (Updater) // updater settings, e.g., SGD + cluster = (Cluster) // cluster settings + optional + loss = (string) // name of loss function type + topk = (int) // the number of results considered to compute accuracy + ''' + assert optimizer != None, 'optimizer (Updater) should be set' + assert cluster != None, 'cluster (Cluster) should be set' + setval(self.jobconf, updater=optimizer.proto) + setval(self.jobconf, cluster=cluster.proto) + + # take care of loss function layer + assert loss != None, 'loss should be set' + if hasattr(self.layers[len(self.layers)-1], 'mask'): + ly = self.layers[len(self.layers)-1].mask + else: + ly = self.layers[len(self.layers)-1].layer + + if ly.type == enumLayerType('softmax'): + # revise the last layer + if loss == 'categorical_crossentropy': + setval(ly, type=enumLayerType('softmaxloss')) + setval(ly.softmaxloss_conf, topk=topk) + elif loss == 'mean_squared_error': + setval(ly, type=enumLayerType('euclideanloss')) + else: + # add new layer + if loss == 'categorical_crossentropy': + self.add(Activation('softmaxloss', topk=topk)) + elif loss == 'mean_squared_error': + self.add(Activation('euclideanloss')) + elif loss == 'user_loss_rnnlm': # user-defined loss layer for rnnlm + self.add(UserLossRNNLM(nclass=kwargs['nclass'], vocab_size=kwargs['in_dim'])) + + def build(self): + net = NetProto() + slyname = self.layers[0].layer.name + for i in range(len(self.layers)): + ly = net.layer.add() + ly.CopyFrom(self.layers[i].layer) + lastly = ly + if self.layers[i].is_datalayer == True: + continue + getattr(ly, 'srclayers').append(slyname) + slyname = ly.name + if hasattr(self.layers[i], 'mask'): + mly = net.layer.add() + mly.CopyFrom(self.layers[i].mask) + getattr(mly, 'srclayers').append(slyname) + slyname = mly.name + lastly = mly + + if self.label == True: + label_layer = Layer(name='label', type=kLabel) + ly = net.layer.add() + ly.CopyFrom(label_layer.layer) + getattr(ly, 'srclayers').append(self.layers[0].layer.name) + getattr(lastly, 'srclayers').append(label_layer.layer.name) + else: + getattr(lastly, 'srclayers').append(self.layers[0].layer.name) + + setval(self.jobconf, neuralnet=net) + + def fit(self, **kwargs): + pass + + def evaluate(self, **kwargs): + pass + + def display(self): + print text_format.MessageToString(self.jobconf) + + +class Sequential(Model): + def __init__(self, name='my model', label=False): + super(Sequential, self).__init__(name=name, label=label) + + def exist_datalayer(self, phase): + if self.layers[0].layer.include != enumPhase(phase): # train + if self.layers[1].layer.include != enumPhase(phase): # test + if self.layers[2].layer.include != enumPhase(phase): # valid + return False + return True + + def fit(self, data=None, train_steps=0, **fields): + ''' + required + data = (Data) // Data class object for training data + train_steps = (int) // the number of training, i.e., epoch + optional + **fields (KEY=VALUE) + batch_size = (int) // batch size for training data + disp_freq = (int) // frequency to display training info + disp_after = (int) // display after this number + validate_data = (Data) // validation data, specified in load_data() + validate_freq = (int) // frequency of validation + validate_steps = (int) // total number of steps for validation + validate_after = (int) // start validation after this number + checkpoint_path = (string) // path to checkpoint file + checkpoint_freq = (int) // frequency for checkpoint + checkpoint_after = (int) // start checkpointing after this number + ''' + assert data != None, 'Training data shold be set' + assert train_steps != 0, 'Training steps shold be set' + + if 'batch_size' in fields: # if new value is set, replace the batch_size + setval(data.layer.store_conf, batchsize=fields['batch_size']) + + if self.exist_datalayer('train') == False: + self.layers.insert(0, data) + + if 'validate_data' in fields: + self.layers.insert(1, fields['validate_data']) + + setval(self.jobconf, train_steps=train_steps) + setval(self.jobconf, **fields) + + # set Train_one_batch component, using backprogapation + setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType('bp')).proto) + + def evaluate(self, data=None, execpath='', **fields): + ''' + required + data = (Data) // Data class object for testing data + optional + test_steps = (int) // the number of testing + **fields (KEY=VALUE) + batch_size = (int) // batch size for testing data + test_freq = (int) // frequency of testing + test_steps = (int) // total number of steps for testing + test_after = (int) // start testing after this number of steps + ''' + assert data != None, 'Testing data shold be set' + + if 'batch_size' in fields: # if new value is set, replace the batch_size + setval(data.layer.store_conf, batchsize=fields['batch_size']) + + if self.exist_datalayer('test') == False: + self.layers.insert(1, data) + + setval(self.jobconf, **fields) + + self.build() # construct Nneuralnet Component + + with open('job.conf', 'w') as f: + f.write(text_format.MessageToString(self.jobconf)) + + #self.display() + self.result = SingaRun(execpath=execpath) + return self.result + +class Store(object): + def __init__(self, **kwargs): + ''' + **kwargs + path = (string) // path to dataset + backend = (string) // + batch_size = (int) // batch size of dataset + shape = (int) // + + ''' + self.proto = Message('Store', **kwargs).proto + +class Algorithm(object): + def __init__(self, type=enumAlgType('bp'), **kwargs): + alg = Message('Alg', alg=type, **kwargs).proto + if type == enumAlgType('cd'): + setval(alg.cd_conf, **kwargs) + self.proto = alg + +class SGD(object): + def __init__(self, lr=0.01, lr_type=None, + decay=0, momentum=0, + step=(0), step_lr=(0.01), **fields): + ''' + required + lr = (float) // base learning rate + lr_type = (string) // type of learning rate + optional + **fields (KEY=VALUE) + decay = (float) // weight decay + momentum = (float) // momentum + + ''' + assert lr + assert lr_type != None, 'type of learning rate should be specified' + + upd = Message('Updater', type=kSGD, **fields).proto + setval(upd.learning_rate, base_lr=lr) + if decay > 0: + setval(upd, weight_decay=decay) + if momentum > 0: + setval(upd, momentum=momentum) + + if lr_type == 'step': + cp = Message('Step', change_freq=60, gamma=0.997) + setval(upd.learning_rate, type=kStep, step_conf=cp.proto) + elif lr_type == 'fixed': + cp = Message('FixedStep', step=step, step_lr=step_lr) + setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto) + elif lr_type == 'linear': + cp = Message('Linear', change_freq=10, final_lr=0.1) + setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto) + self.proto = upd + + +class Cluster(object): + def __init__(self, workspace=None, + nworker_groups=1, nserver_groups=1, + nworkers_per_group=1, nservers_per_group=1, + nworkers_per_procs=1, nservers_per_procs=1, + **fields): + ''' + required + workspace = (string) // workspace path + optional + nworker_groups = (int) + nserver_groups = (int) + nworkers_per_group = (int) + nservers_per_group = (int) + nworkers_per_procs = (int) + nservers_per_procs = (int) + **fields + server_worker_separate = (bool) + ''' + assert workspace != None, 'need to set workspace' + self.proto = Message('Cluster', workspace=workspace).proto + # optional + self.proto.nworker_groups = nworker_groups + self.proto.nserver_groups = nserver_groups + self.proto.nworkers_per_group = nworkers_per_group + self.proto.nservers_per_group = nservers_per_group + self.proto.nworkers_per_procs = nworkers_per_procs + self.proto.nservers_per_procs = nservers_per_procs + # other fields + setval(self.proto, **fields) + +#TODO make this internally used +class Parameter(object): + + def __init__(self, **kwargs): + ''' + optional + **kwargs + name = (string) // parameter name + lr = (float) // learning rate + wd = (float) // weight decay + init = (string) // initialized type {'constant','uniform','gaussian'} + value = (int) // value for 'constant' + range = (float) // [low, high] for 'uniform', low=-range, high=range + low = (float) // low value for 'uniform' + high = (float) // high value for 'uniform' + mean = (float) // mean for 'gaussian' + std = (float) // std for 'gaussian' + ''' + fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1, + 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1 + } + self.param = Message('Param', **fields).proto + + if not 'name' in kwargs: + setval(self.param, name=generateName('param', 1)) + else: + setval(self.param, name=kwargs['name']) + + if 'range' in kwargs: + kwargs['low'] = -float(kwargs['range']) + kwargs['high'] = float(kwargs['range']) + + if 'init' in kwargs: + pg = Message('ParamGen', type=enumInitMethod(kwargs['init']), **kwargs) + del kwargs['init'] + else: # default: uniform + pg = Message('ParamGen', type=enumInitMethod(kwargs['uniform'])) + setval(self.param, init=pg.proto) + + + #if 'param_init' in kwargs: + # setval(self.param, init=kwargs['param_init'].proto) + # del kwargs['param_init'] + + def update(self, **fields): + setval(self.param, **fields) + setval(self.param.init, **fields) + + +class Layer(object): + def __init__(self, **kwargs): + self.layer = Message('Layer', **kwargs).proto + # required + if not 'name' in kwargs: + setval(self.layer, name=generateName('layer', 1)) + + # srclayers are set in Model.build() + self.is_datalayer = False + + def setParamField(self, param, pname, **kwargs): + # param: ParamProto + if pname == 'w': + field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else param.lr_scale, + 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else param.wd_scale + } + elif pname == 'b': + field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else param.lr_scale, + 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else param.wd_scale + } + setval(param, name=generateName(pname), **field) + +class Data(Layer): + def __init__(self, load, phase='train', conf=None, **kwargs): + assert load != None, 'data type should be specified' + if load == 'kData': + super(Data, self).__init__(name=generateName('data'), user_type=load) + else: + self.layer_type = enumLayerType(load) + super(Data, self).__init__(name=generateName('data'), type=self.layer_type) + + # include/exclude + setval(self.layer, include=enumPhase(phase)) + #setval(self.layer, exclude=kTest if phase=='train' else kTrain) + + if conf == None: + if load == 'kData': + setval(self.layer.Extensions[data_conf], **kwargs) + else: + setval(self.layer.store_conf, **kwargs) + else: + setval(self.layer, store_conf=conf.proto) + self.is_datalayer = True + +class Convolution2D(Layer): + def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0, + init='uniform', w_param=None, b_param=None, + activation=None, **kwargs): + ''' + required + nb_filter = (int) // the number of filters + kernel = (int) // the size of filter + optional + stride = (int) // the size of stride + pad = (int) // the size of padding + xxxxx + + ''' + assert nb_filter > 0 and kernel > 0, 'should be set as positive int' + super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution) + fields = {'num_filters' : nb_filter, + 'kernel' : kernel, + 'stride' : stride, + 'pad' : pad} + setval(self.layer.convolution_conf, **fields) + + # parameter w + if w_param == None: + w_param = Parameter(name=generateName('w'), init=init) # default: uniform + else: + field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else w_param.param.lr_scale, + 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else w_param.param.wd_scale + } + setval(w_param.param, name=generateName('w'), **field) + setval(w_param.param.init, **kwargs) + setval(self.layer, param=w_param.param) + + # parameter b + if b_param == None: + b_param = Parameter(name=generateName('b'), init=init) # default: uniform + else: + field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else b_param.param.lr_scale, + 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else b_param.param.wd_scale + } + setval(b_param.param, name=generateName('b'), **field) + setval(self.layer, param=b_param.param) + + # following layers: e.g., activation, dropout, etc. + if activation: + self.mask = Activation(activation=activation).layer + +class MaxPooling2D(Layer): + def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): + ''' + required + pool_size = (int|tuple) // the size for pooling + optional + stride = (int) // the size of striding + ignore_border = (bool) // flag for padding + **kwargs // fields for Layer class + ''' + assert pool_size != None, 'pool_size is required' + if type(pool_size) == int: + pool_size = (pool_size, pool_size) + assert type(pool_size) == tuple and \ + pool_size[0] == pool_size[1], 'pool size should be square in Singa' + super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) + fields = {'pool' : PoolingProto().MAX, + 'kernel' : pool_size[0], + 'stride' : stride, + 'pad' : 0 if ignore_border else 1} + setval(self.layer.pooling_conf, **fields) + +class AvgPooling2D(Layer): + def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): + ''' + required + pool_size = (int|tuple) // size for pooling + optional + stride = (int) // size of striding + ignore_border = (bool) // flag for padding + **kwargs // fields for Layer class + ''' + assert pool_size != None, 'pool_size is required' + if type(pool_size) == int: + pool_size = (pool_size, pool_size) + assert type(pool_size) == tuple and \ + pool_size[0] == pool_size[1], 'pool size should be square in Singa' + super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) + self.layer.pooling_conf.pool = PoolingProto().AVG + fields = {'pool' : PoolingProto().AVG, + 'kernel' : pool_size[0], + 'stride' : stride, + 'pad' : 0 if ignore_border else 1} + setval(self.layer.pooling_conf, **fields) + +class LRN2D(Layer): + def __init__(self, size=0, alpha=1e-4, k=1, beta=0.75, **kwargs): + super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN) + # required + assert size != 0, 'local size should be set' + self.layer.lrn_conf.local_size = size + setval(self.layer.lrn_conf, alpha=alpha, knorm=k, beta=beta, **kwargs) + +class Dense(Layer): + def __init__(self, output_dim=0, activation=None, + init='uniform', w_param=None, b_param=None, input_dim=None, + **kwargs): + ''' + required + output_dim = (int) + optional + activation = (string) + **kwargs + w_lr + w_wd + + ''' + assert output_dim > 0, 'output_dim should be set' + super(Dense, self).__init__(type=kInnerProduct, **kwargs) + self.layer.innerproduct_conf.num_output = output_dim # required + + # parameter w + if w_param == None: + w_param = Parameter(name=generateName('w'), init=init) # default: uniform + else: + self.setParamField(w_param.param, 'w', **kwargs) + setval(self.layer, param=w_param.param) + + # parameter b + if b_param == None: + b_param = Parameter(name=generateName('b'), init=init) # default: uniform + else: + self.setParamField(b_param.param, 'b', **kwargs) + setval(self.layer, param=b_param.param) + + # following layers: e.g., activation, dropout, etc. + if activation: + self.mask = Activation(activation=activation).layer + +class Activation(Layer): + def __init__(self, activation='stanh', topk=1): + self.name = activation + #TODO better way? + if activation == 'tanh': activation = 'stanh' + self.layer_type = enumLayerType(activation) + super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type) + if activation == 'softmaxloss': + self.layer.softmaxloss_conf.topk = topk + +class Dropout(Layer): + def __init__(self, ratio=0.5): + self.name = 'dropout' + self.layer_type = kDropout + super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type) + self.layer.dropout_conf.dropout_ratio = ratio + + +class RGB(Layer): + def __init__(self, meanfile=None, **kwargs): + assert meanfile != None, 'meanfile should be specified' + self.name = 'rgb' + self.layer_type = kRGBImage + super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type) + self.layer.rgbimage_conf.meanfile = meanfile + +class Embedding(Layer): + def __init__(self, in_dim, out_dim, w_param=None, **kwargs): + super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding') + fields = { 'vocab_size': in_dim, + 'word_dim': out_dim } + setval(self.layer.Extensions[embedding_conf], **fields) + if w_param == None: + w_param = Parameter(name=generateName('w'), init=init) # default: uniform + else: + self.setParamField(w_param.param, 'w', **kwargs) + setval(self.layer, param=w_param.param) + +class RNNLM(Layer): + def __init__(self, dim, w_param=None, **kwargs): + super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden') + if w_param == None: + w_param = Parameter(name=generateName('w'), init=init) # default: uniform + else: + self.setParamField(w_param.param, 'w', **kwargs) + setval(self.layer, param=w_param.param) + +class UserLossRNNLM(Layer): + def __init__(self, **kwargs): + super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss') + self.layer.Extensions[loss_conf].nclass = kwargs['nclass'] + self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size'] + setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', range=0.3).param) + setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', range=0.3).param) + + +#TODO run singa training/testing via a wrapper for Driver +def SingaRun(execpath=''): + SINGAROOT = '../../' + conf = 'job.conf' + if execpath=='': + cmd = '../../bin/singa-run.sh ' \ + + '-conf %s ' % conf + else: + cmd = '../../bin/singa-run.sh ' \ + + '-conf %s ' % conf \ + + '-exec %s ' % execpath + + procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + + resultDic = {} + output = iter(procs.stdout.readline, '') + for line in output: + print line[:-1] + line = re.findall(r'[\w|*.*]+', line) + if 'accuracy' in line: + step = line[line.index('step')+1] + acc = line[line.index('accuracy')+1] + loss = line[line.index('loss')+1] + #print 'Step: ', line[idx_step+1], 'Acc: ', acc, 'Loss: ', loss + resultDic.setdefault(step,{})['acc'] = acc + resultDic.setdefault(step,{})['loss'] = loss + elif 'Train' in line: + step = line[line.index('step')+1] + loss = line[line.index('loss')+1] + ppl = line[line.index('ppl')+1] + resultDic.setdefault(step,{})['loss'] = loss + resultDic.setdefault(step,{})['ppl'] = ppl + + #TODO better format to store?? + return resultDic + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/singa/__init__.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/__init__.py b/tool/python/singa/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/singa/driver.i ---------------------------------------------------------------------- diff --git a/tool/python/singa/driver.i b/tool/python/singa/driver.i new file mode 100644 index 0000000..d1154a8 --- /dev/null +++ b/tool/python/singa/driver.i @@ -0,0 +1,18 @@ +%module driver +%include "std_vector.i" +%include "std_string.i" +%include "argcargv.i" +%apply (int ARGC, char **ARGV) { (int argc, char **argv) } +%{ +#include "../include/singa/driver.h" +%} + +namespace singa{ +using std::vector; +class Driver{ +public: +void Train(bool resume, const std::string job_conf); +void Init(int argc, char **argv); +}; +} + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b8104eff/tool/python/singa/driver.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/driver.py b/tool/python/singa/driver.py new file mode 100644 index 0000000..f84df9c --- /dev/null +++ b/tool/python/singa/driver.py @@ -0,0 +1,120 @@ +# This file was automatically generated by SWIG (http://www.swig.org). +# Version 3.0.2 +# +# Do not make changes to this file unless you know what you are doing--modify +# the SWIG interface file instead. + + + + + +from sys import version_info +if version_info >= (2,6,0): + def swig_import_helper(): + from os.path import dirname + import imp + fp = None + try: + fp, pathname, description = imp.find_module('_driver', [dirname(__file__)]) + except ImportError: + import _driver + return _driver + if fp is not None: + try: + _mod = imp.load_module('_driver', fp, pathname, description) + finally: + fp.close() + return _mod + _driver = swig_import_helper() + del swig_import_helper +else: + import _driver +del version_info +try: + _swig_property = property +except NameError: + pass # Python < 2.2 doesn't have 'property'. +def _swig_setattr_nondynamic(self,class_type,name,value,static=1): + if (name == "thisown"): return self.this.own(value) + if (name == "this"): + if type(value).__name__ == 'SwigPyObject': + self.__dict__[name] = value + return + method = class_type.__swig_setmethods__.get(name,None) + if method: return method(self,value) + if (not static): + self.__dict__[name] = value + else: + raise AttributeError("You cannot add attributes to %s" % self) + +def _swig_setattr(self,class_type,name,value): + return _swig_setattr_nondynamic(self,class_type,name,value,0) + +def _swig_getattr(self,class_type,name): + if (name == "thisown"): return self.this.own() + method = class_type.__swig_getmethods__.get(name,None) + if method: return method(self) + raise AttributeError(name) + +def _swig_repr(self): + try: strthis = "proxy of " + self.this.__repr__() + except: strthis = "" + return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) + +try: + _object = object + _newclass = 1 +except AttributeError: + class _object : pass + _newclass = 0 + + +class SwigPyIterator(_object): + __swig_setmethods__ = {} + __setattr__ = lambda self, name, value: _swig_setattr(self, SwigPyIterator, name, value) + __swig_getmethods__ = {} + __getattr__ = lambda self, name: _swig_getattr(self, SwigPyIterator, name) + def __init__(self, *args, **kwargs): raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _driver.delete_SwigPyIterator + __del__ = lambda self : None; + def value(self): return _driver.SwigPyIterator_value(self) + def incr(self, n=1): return _driver.SwigPyIterator_incr(self, n) + def decr(self, n=1): return _driver.SwigPyIterator_decr(self, n) + def distance(self, *args): return _driver.SwigPyIterator_distance(self, *args) + def equal(self, *args): return _driver.SwigPyIterator_equal(self, *args) + def copy(self): return _driver.SwigPyIterator_copy(self) + def next(self): return _driver.SwigPyIterator_next(self) + def __next__(self): return _driver.SwigPyIterator___next__(self) + def previous(self): return _driver.SwigPyIterator_previous(self) + def advance(self, *args): return _driver.SwigPyIterator_advance(self, *args) + def __eq__(self, *args): return _driver.SwigPyIterator___eq__(self, *args) + def __ne__(self, *args): return _driver.SwigPyIterator___ne__(self, *args) + def __iadd__(self, *args): return _driver.SwigPyIterator___iadd__(self, *args) + def __isub__(self, *args): return _driver.SwigPyIterator___isub__(self, *args) + def __add__(self, *args): return _driver.SwigPyIterator___add__(self, *args) + def __sub__(self, *args): return _driver.SwigPyIterator___sub__(self, *args) + def __iter__(self): return self +SwigPyIterator_swigregister = _driver.SwigPyIterator_swigregister +SwigPyIterator_swigregister(SwigPyIterator) + +class Driver(_object): + __swig_setmethods__ = {} + __setattr__ = lambda self, name, value: _swig_setattr(self, Driver, name, value) + __swig_getmethods__ = {} + __getattr__ = lambda self, name: _swig_getattr(self, Driver, name) + __repr__ = _swig_repr + def Train(self, *args): return _driver.Driver_Train(self, *args) + def Init(self, *args): return _driver.Driver_Init(self, *args) + def __init__(self): + this = _driver.new_Driver() + try: self.this.append(this) + except: self.this = this + __swig_destroy__ = _driver.delete_Driver + __del__ = lambda self : None; +Driver_swigregister = _driver.Driver_swigregister +Driver_swigregister(Driver) + +# This file is compatible with both classic and new-style classes. + +
