SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
- Update model.py to take care of cudnn . fit() receives device field, a list of gpu id . setCudnnLayerType converts LayerType to CdunnLayerType - Add examples for cudnn Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ceb06656 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ceb06656 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ceb06656 Branch: refs/heads/master Commit: ceb066560708a6b5e4cbdbbcba095821f18db327 Parents: 7d43e27 Author: chonho <[email protected]> Authored: Mon Dec 28 13:31:30 2015 +0800 Committer: chonho <[email protected]> Committed: Fri Jan 1 15:59:14 2016 +0800 ---------------------------------------------------------------------- tool/python/examples/cifar10_cnn_cudnn.py | 34 +++++++++++++++ tool/python/examples/mnist_mlp_cudnn.py | 34 +++++++++++++++ tool/python/singa/driver.py | 2 +- tool/python/singa/driver_wrap.cxx | 57 ++++++++++++++------------ tool/python/singa/model.py | 51 +++++++++++++++++++---- 5 files changed, 144 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/examples/cifar10_cnn_cudnn.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py new file mode 100755 index 0000000..e08610a --- /dev/null +++ b/tool/python/examples/cifar10_cnn_cudnn.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__),'..')) +from singa.model import * +from singa.datasets import cifar10 + +X_train, X_test, workspace = cifar10.load_data() + +m = Sequential('cifar10-cnn', sys.argv) + +m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2)) +m.add(MaxPooling2D(pool_size=(3,3), stride=2)) +m.add(Activation('relu')) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution2D(32, 5, 1, 2, b_lr=2)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution2D(64, 5, 1, 2)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) + +m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax')) + +sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) +topo = Cluster(workspace) +m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) + +gpu_id = [0] +m.fit(X_train, nb_epoch=1000, with_test=True, device=gpu_id) +result = m.evaluate(X_test, test_steps=100, test_freq=300) + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/examples/mnist_mlp_cudnn.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_mlp_cudnn.py b/tool/python/examples/mnist_mlp_cudnn.py new file mode 100755 index 0000000..d418950 --- /dev/null +++ b/tool/python/examples/mnist_mlp_cudnn.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__),'..')) +from singa.model import * +from singa.datasets import mnist + +# Sample parameter values for Mnist MLP example +pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000, + 'std_value' : 127.5, 'mean_value' : 127.5} +X_train, X_test, workspace = mnist.load_data(**pvalues) + +m = Sequential('mlp', argv=sys.argv) + +''' Weight and Bias are initialized by + uniform distribution with scale=0.05 at default +''' +m.add(Dense(2500, init='uniform', activation='tanh')) +m.add(Dense(2000, init='uniform', activation='tanh')) +m.add(Dense(1500, init='uniform', activation='tanh')) +m.add(Dense(1000, init='uniform', activation='tanh')) +m.add(Dense(500, init='uniform', activation='tanh')) +m.add(Dense(10, init='uniform', activation='softmax')) + +sgd = SGD(lr=0.001, lr_type='step') +topo = Cluster(workspace) +m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) + +gpu_id = [0] +m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id) +result = m.evaluate(X_test, batch_size=100, test_steps=10) + +#e.g., display result +#for k, v in sorted(result.items(), key=lambda x: x[0]): +# print k, v http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/driver.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/driver.py b/tool/python/singa/driver.py index c1aac7f..d203923 100644 --- a/tool/python/singa/driver.py +++ b/tool/python/singa/driver.py @@ -1,5 +1,5 @@ # This file was automatically generated by SWIG (http://www.swig.org). -# Version 3.0.2 +# Version 2.0.11 # # Do not make changes to this file unless you know what you are doing--modify # the SWIG interface file instead. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/driver_wrap.cxx ---------------------------------------------------------------------- diff --git a/tool/python/singa/driver_wrap.cxx b/tool/python/singa/driver_wrap.cxx index e6e6de2..40ea5ce 100644 --- a/tool/python/singa/driver_wrap.cxx +++ b/tool/python/singa/driver_wrap.cxx @@ -1,6 +1,6 @@ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). - * Version 3.0.2 + * Version 2.0.11 * * This file is not intended to be easily readable and contains a number of * coding conventions designed to improve portability and efficiency. Do not make @@ -560,14 +560,14 @@ SWIG_MangledTypeQueryModule(swig_module_info *start, swig_module_info *iter = start; do { if (iter->size) { - size_t l = 0; - size_t r = iter->size - 1; + register size_t l = 0; + register size_t r = iter->size - 1; do { /* since l+r >= 0, we can (>> 1) instead (/ 2) */ - size_t i = (l + r) >> 1; + register size_t i = (l + r) >> 1; const char *iname = iter->types[i]->name; if (iname) { - int compare = strcmp(name, iname); + register int compare = strcmp(name, iname); if (compare == 0) { return iter->types[i]; } else if (compare < 0) { @@ -611,7 +611,7 @@ SWIG_TypeQueryModule(swig_module_info *start, of the str field (the human readable name) */ swig_module_info *iter = start; do { - size_t i = 0; + register size_t i = 0; for (; i < iter->size; ++i) { if (iter->types[i]->str && (SWIG_TypeEquiv(iter->types[i]->str, name))) return iter->types[i]; @@ -630,10 +630,10 @@ SWIG_TypeQueryModule(swig_module_info *start, SWIGRUNTIME char * SWIG_PackData(char *c, void *ptr, size_t sz) { static const char hex[17] = "0123456789abcdef"; - const unsigned char *u = (unsigned char *) ptr; - const unsigned char *eu = u + sz; + register const unsigned char *u = (unsigned char *) ptr; + register const unsigned char *eu = u + sz; for (; u != eu; ++u) { - unsigned char uu = *u; + register unsigned char uu = *u; *(c++) = hex[(uu & 0xf0) >> 4]; *(c++) = hex[uu & 0xf]; } @@ -645,11 +645,11 @@ SWIG_PackData(char *c, void *ptr, size_t sz) { */ SWIGRUNTIME const char * SWIG_UnpackData(const char *c, void *ptr, size_t sz) { - unsigned char *u = (unsigned char *) ptr; - const unsigned char *eu = u + sz; + register unsigned char *u = (unsigned char *) ptr; + register const unsigned char *eu = u + sz; for (; u != eu; ++u) { - char d = *(c++); - unsigned char uu; + register char d = *(c++); + register unsigned char uu; if ((d >= '0') && (d <= '9')) uu = ((d - '0') << 4); else if ((d >= 'a') && (d <= 'f')) @@ -1326,7 +1326,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi } if (!PyTuple_Check(args)) { if (min <= 1 && max >= 1) { - int i; + register int i; objs[0] = args; for (i = 1; i < max; ++i) { objs[i] = 0; @@ -1336,7 +1336,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi PyErr_SetString(PyExc_SystemError, "UnpackTuple() argument list is not a tuple"); return 0; } else { - Py_ssize_t l = PyTuple_GET_SIZE(args); + register Py_ssize_t l = PyTuple_GET_SIZE(args); if (l < min) { PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", name, (min == max ? "" : "at least "), (int)min, (int)l); @@ -1346,7 +1346,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi name, (min == max ? "" : "at most "), (int)max, (int)l); return 0; } else { - int i; + register int i; for (i = 0; i < l; ++i) { objs[i] = PyTuple_GET_ITEM(args, i); } @@ -2461,7 +2461,7 @@ SWIG_Python_NewShadowInstance(SwigPyClientData *data, PyObject *swig_this) } } else { #if PY_VERSION_HEX >= 0x03000000 - inst = ((PyTypeObject*) data->newargs)->tp_new((PyTypeObject*) data->newargs, Py_None, Py_None); + inst = PyBaseObject_Type.tp_new((PyTypeObject*) data->newargs, Py_None, Py_None); if (inst) { PyObject_SetAttr(inst, SWIG_This(), swig_this); Py_TYPE(inst)->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; @@ -2967,7 +2967,7 @@ static swig_module_info swig_module = {swig_types, 5, 0, 0, 0, 0}; #endif #define SWIG_name "_driver" -#define SWIGVERSION 0x030002 +#define SWIGVERSION 0x020011 #define SWIG_VERSION SWIGVERSION @@ -3060,9 +3060,11 @@ namespace swig { #include <string> +#include <stdexcept> +#include <stddef.h> -#include <stddef.h> + #include <stddef.h> namespace swig { @@ -3407,22 +3409,25 @@ SWIG_AsVal_ptrdiff_t (PyObject * obj, ptrdiff_t *val) } +#include <stdexcept> + + #include <algorithm> #include <vector> +#include <string> + + #include "singa/driver.h" SWIGINTERN int SWIG_AsVal_bool (PyObject *obj, bool *val) { - int r; - if (!PyBool_Check(obj)) - return SWIG_ERROR; - r = PyObject_IsTrue(obj); + int r = PyObject_IsTrue(obj); if (r == -1) return SWIG_ERROR; if (val) *val = r ? true : false; @@ -4735,7 +4740,7 @@ static swig_const_info swig_const_table[] = { * array with the correct data and linking the correct swig_cast_info * structures together. * - * The generated swig_type_info structures are assigned statically to an initial + * The generated swig_type_info structures are assigned staticly to an initial * array. We just loop through that array, and handle each type individually. * First we lookup if this type has been already loaded, and if so, use the * loaded structure instead of the generated one. Then we have to fill in the @@ -5065,7 +5070,7 @@ extern "C" { var = var->next; } if (res == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + PyErr_SetString(PyExc_NameError,"Unknown C global variable"); } return res; } @@ -5082,7 +5087,7 @@ extern "C" { var = var->next; } if (res == 1 && !PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + PyErr_SetString(PyExc_NameError,"Unknown C global variable"); } return res; } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/model.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py index 1aa62b0..ade0b85 100644 --- a/tool/python/singa/model.py +++ b/tool/python/singa/model.py @@ -19,6 +19,7 @@ class Model(object): self.argv = argv self.result = None self.last_checkpoint_path = None + self.cudnn = False def exist_datalayer(self, phase): for ly in self.layers: @@ -104,18 +105,23 @@ class Model(object): else: getattr(lastly, 'srclayers').append(self.layers[0].layer.name) + # use of cudnn + if self.cudnn == True: + self.setCudnnLayerType(net) + setval(self.jobconf, neuralnet=net) def fit(self, data=None, alg='bp', nb_epoch=0, - with_test=False, execpath='', **fields): + with_test=False, execpath='', device=None, **fields): ''' required - data = (Data) // Data class object for training data - alg = (string) // algorithm, e.g., 'bp', 'cd' - nb_epoch = (int) // the number of training steps + data = (Data) // Data class object for training data + alg = (string) // algorithm, e.g., 'bp', 'cd' + nb_epoch = (int) // the number of training steps optional - with_test = (bool) // flag if singa runs for test data - execpath = (string) // path to user own singa (executable file) + with_test = (bool) // flag if singa runs for test data + execpath = (string) // path to user own singa (executable file) + device = (int/list) // a list of gpu ids **fields (KEY=VALUE) batch_size = (int) // batch size for training data train_steps = (int) // the number of steps for training, i.e., epoch @@ -161,6 +167,11 @@ class Model(object): # set Train_one_batch component, using backprogapation at default setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto) + # use of cudnn + if device != None: + setval(self.jobconf, gpu=device) + self.cudnn = True + # start to run singa for training if with_test == False: self.build() # construct Nneuralnet Component @@ -216,7 +227,7 @@ class Model(object): #filename = 'job.conf' #with open(filename, 'w') as f: # f.write(text_format.MessageToString(self.jobconf.cluster)) - #self.display() + self.display() #--- run singa --- return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly) @@ -224,8 +235,32 @@ class Model(object): def display(self): + ''' print out job proto + ''' print text_format.MessageToString(self.jobconf) + def setCudnnLayerType(self, net): + ''' convert LayerType to CdunnLayerType + ''' + for i in range(len(net.layer)): + ly_type = net.layer[i].type + cudnn_ly_type = ly_type + if ly_type == kCConvolution: cudnn_ly_type = kCudnnConv + elif ly_type == kCPooling: cudnn_ly_type = kCudnnPool + elif ly_type == kLRN: cudnn_ly_type = kCudnnLRN + elif ly_type == kSoftmax: cudnn_ly_type = kCudnnSoftmax + elif ly_type == kSoftmaxLoss: cudnn_ly_type = kCudnnSoftmaxLoss + elif ly_type == kSTanh: + cudnn_ly_type = kCudnnActivation + net.layer[i].activation_conf.type = STANH + elif ly_type == kSigmoid: + cudnn_ly_type = kCudnnActivation + net.layer[i].activation_conf.type = SIGMOID + elif ly_type == kReLU: + cudnn_ly_type = kCudnnActivation + net.layer[i].activation_conf.type = RELU + net.layer[i].type = cudnn_ly_type + class Energy(Model): def __init__(self, name='my model', argv=[], label=False): @@ -277,6 +312,8 @@ class Sequential(Model): if i == len(layer.hid_dim)+1: dim = layer.out_dim else: dim = layer.hid_dim[i-1] self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation)) + else: + self.layers.append(layer) else: self.layers.append(layer)
