Repository: incubator-singa Updated Branches: refs/heads/master 8b7d1e09e -> c97b970dc
SINGA-126 Python Binding for Interactive Training - revise Feed() function in dummy.cc . it now takes shape, data for input, aux_data for label . revised neuron_layer.h, driver.i - add ImageInput() and LabelInput() . inherited from Dummy() . specific to image data and label data, respectively - add compute_mean_image() method in train_cifar10.py . it loads 5 cifar10 datasets and computes the average image - add Update() in model.py . it updates layer parameters Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5a8fc373 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5a8fc373 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5a8fc373 Branch: refs/heads/master Commit: 5a8fc3732e00372493ffdb1422db56da7f6e4876 Parents: 8130b7e Author: chonho <[email protected]> Authored: Sun Apr 3 03:06:28 2016 +0800 Committer: chonho <[email protected]> Committed: Tue Apr 5 23:22:51 2016 +0800 ---------------------------------------------------------------------- include/singa/neuralnet/neuron_layer.h | 2 +- src/neuralnet/neuron_layer/dummy.cc | 10 +- tool/python/examples/train_cifar10.py | 89 +++++++----- tool/python/examples/train_mnist.py | 20 ++- tool/python/singa/driver.i | 2 +- tool/python/singa/layer.py | 207 +++++++--------------------- tool/python/singa/model.py | 16 +++ tool/python/singa/utils/utility.py | 5 - 8 files changed, 146 insertions(+), 205 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/include/singa/neuralnet/neuron_layer.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h index 2d73854..4dab469 100644 --- a/include/singa/neuralnet/neuron_layer.h +++ b/include/singa/neuralnet/neuron_layer.h @@ -126,7 +126,7 @@ class DummyLayer: public NeuronLayer { void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; - void Feed(int batchsize, vector<float>& data, int is_aux); + void Feed(int batchsize, vector<float>& data, vector<int>& aux_data); Layer* ToLayer() { return this;} private: http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/src/neuralnet/neuron_layer/dummy.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc index 001e053..9796407 100644 --- a/src/neuralnet/neuron_layer/dummy.cc +++ b/src/neuralnet/neuron_layer/dummy.cc @@ -78,22 +78,22 @@ void DummyLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { Copy(grad_, srclayers[0]->mutable_grad(this)); } -void DummyLayer::Feed(int batchsize, vector<float>& data, int is_aux){ +void DummyLayer::Feed(int batchsize, vector<float>& data, vector<int>& aux_data){ batchsize_ = batchsize; // input data - if (is_aux == 0) { + if (data.size() > 0) { int size = data.size(); float* ptr = data_.mutable_cpu_data(); for (int i = 0; i< size; i++) { ptr[i] = data.at(i); } } - // label - else { + // auxiliary data, e.g., label + if (aux_data.size() > 0) { aux_data_.resize(batchsize_); for (int i = 0; i< batchsize_; i++) { - aux_data_[i] = static_cast<int>(data.at(i)); + aux_data_[i] = static_cast<int>(aux_data.at(i)); } } return; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/examples/train_cifar10.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/train_cifar10.py b/tool/python/examples/train_cifar10.py index a757595..6c621a3 100755 --- a/tool/python/examples/train_cifar10.py +++ b/tool/python/examples/train_cifar10.py @@ -28,52 +28,73 @@ import os, sys import numpy as np current_path_ = os.path.dirname(__file__) -singa_root_=os.path.abspath(os.path.join(current_path_,'../../..')) +singa_root_ = os.path.abspath(os.path.join(current_path_,'../../..')) sys.path.append(os.path.join(singa_root_,'tool','python')) from singa.driver import Driver from singa.layer import * from singa.model import * -from singa.utils.utility import swap32 -fname_mean_image = 'tool/python/examples/datasets/cifar10_mean_image' -mean_image = np.fromfile(fname_mean_image) + +''' +CIFAR10 dataset can be downloaded at [https://www.cs.toronto.edu/~kriz/cifar.html] +- please specify dataset_dir +''' +dataset_dir_ = singa_root_ + "/tool/python/examples/datasets/cifar-10-batches-py" +mean_image = None + +def unpickle(file): + ''' This method loads dataset provided at CIFAR10 website + See [https://www.cs.toronto.edu/~kriz/cifar.html] for more details + ''' + import cPickle + fo = open(file, 'rb') + dict = cPickle.load(fo) + fo.close() + return dict + +def compute_mean_image(): + ''' This is a sample script to cmopute the average image + of all samples in 5 dataset of cifar10 + ''' + mean = None + nb_samples_total = 0 + for did in range(1,6): + fname_train_data = dataset_dir_ + "/data_batch_{}".format(did) + cifar10 = unpickle(fname_train_data) + image = cifar10['data'].astype(dtype=np.uint8) + if did > 1: + image = np.vstack((image, image)) + return np.average(image, axis=0) def load_dataset(did=1): - ''' CIFAR10 dataset + ''' CIFAR10 dataset includes 5 binary dataset, each contains 10000 images 1 row (1 image) includes 1 label & 3072 pixels 3072 pixels are 3 channels of a 32x32 image ''' - print '[Load CIFAR10 dataset]', did - dataset_dir_ = singa_root_ + "/examples/cifar10/cifar-10-batches-bin" - fname_train_data = dataset_dir_ + "/data_batch_{}.bin".format(did) - - nb_samples = 10000 - nb_pixels = 3 * 32 * 32 - d = np.fromfile(fname_train_data, dtype=np.uint8) - d = d.reshape(nb_samples, nb_pixels + 1) # +1 for label - x = d[:, 1:] - x = x - mean_image - print ' data x:', x.shape - y = d[:, 0] - y = y.reshape(nb_samples, 1) - print ' label y:', y.shape - return x, y - -def get_labellist(): - dataset_dir_ = singa_root_ + "/examples/cifar10/cifar-10-batches-bin" - fname_label_list = dataset_dir_ + "/batches.meta.txt" - label_list_ = np.genfromtxt(fname_label_list, dtype=str) - return label_list_ + assert mean_image != None, 'mean_image is required' + print '[Load CIFAR10 dataset {}]'.format(did) + fname_train_data = dataset_dir_ + "/data_batch_{}".format(did) + cifar10 = unpickle(fname_train_data) + image = cifar10['data'].astype(dtype=np.uint8) + image = image - mean_image + print ' image x:', image.shape + label = np.asarray(cifar10['labels'], dtype=np.uint8) + label = label.reshape(label.size, 1) + print ' label y:', label.shape + return image, label #------------------------------------------------------------------- +mean_image = compute_mean_image() +# mean_image = np.fromfile('tool/python/examples/datasets/cifar10_mean_image') + print '[Layer registration/declaration]' d = Driver() d.Init(sys.argv) -input = Dummy() -label = Dummy() +input = ImageInput(32, 32, 3) # image width, height, channel +label = LabelInput() nn = [] nn.append(input) @@ -99,19 +120,23 @@ batchsize = 100 disp_freq = 50 train_step = 1000 +print '[Start training]' for dataset_id in range(train_step / batchsize): x, y = load_dataset(dataset_id%5+1) - print '[Start training]' for i in range(x.shape[0] / batchsize): xb, yb = x[i*batchsize:(i+1)*batchsize,:], y[i*batchsize:(i+1)*batchsize,:] - nn[0].Feed(xb, 3, 0) - label.Feed(yb, 1, 1) + nn[0].Feed(xb) + label.Feed(yb) for h in range(1, len(nn)): nn[h].ComputeFeature(nn[h-1]) loss.ComputeFeature(nn[-1], label) if (i+1)%disp_freq == 0: print ' Step {:>3}: '.format(i+1 + dataset_id*(x.shape[0]/batchsize)), loss.display() - loss.ComputeGradient(i+1, sgd) + + loss.ComputeGradient() + for h in range(len(nn)-1, 0, -1): + nn[h].ComputeGradient() + sgd.Update(i+1, nn[h]) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/examples/train_mnist.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/train_mnist.py b/tool/python/examples/train_mnist.py index 466bc58..b8e6217 100755 --- a/tool/python/examples/train_mnist.py +++ b/tool/python/examples/train_mnist.py @@ -34,7 +34,12 @@ sys.path.append(os.path.join(singa_root_,'tool','python')) from singa.driver import Driver from singa.layer import * from singa.model import * -from singa.utils.utility import swap32 + +def swap32(x): + return (((x << 24) & 0xFF000000) | + ((x << 8) & 0x00FF0000) | + ((x >> 8) & 0x0000FF00) | + ((x >> 24) & 0x000000FF)) def load_dataset(): ''' MNIST dataset @@ -66,8 +71,8 @@ print '[Layer registration/declaration]' d = Driver() d.Init(sys.argv) -input = Dummy() -label = Dummy() +input = ImageInput(28, 28) +label = LabelInput() nn = [] nn.append(input) @@ -97,11 +102,16 @@ print '[Start training]' for i in range(x.shape[0] / batchsize): xb, yb = x[i*batchsize:(i+1)*batchsize,:], y[i*batchsize:(i+1)*batchsize,:] nn[0].Feed(xb) - label.Feed(yb, is_label=1) + label.Feed(yb) for h in range(1, len(nn)): nn[h].ComputeFeature(nn[h-1]) loss.ComputeFeature(nn[-1], label) if (i+1)%disp_freq == 0: print ' Step {:>3}: '.format(i+1), loss.display() - loss.ComputeGradient(i+1, sgd) + + loss.ComputeGradient() + for h in range(len(nn)-1, 0, -1): + nn[h].ComputeGradient() + sgd.Update(i+1, nn[h]) + http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/driver.i ---------------------------------------------------------------------- diff --git a/tool/python/singa/driver.i b/tool/python/singa/driver.i index 1650145..55cec02 100644 --- a/tool/python/singa/driver.i +++ b/tool/python/singa/driver.i @@ -79,7 +79,7 @@ namespace singa{ class DummyLayer{ public: void Setup(const std::string str, const std::vector<singa::Layer*>& srclayers); - void Feed(int batchsize, std::vector<float>& data, int is_aux); + void Feed(int batchsize, std::vector<float>& data, std::vector<int>& aux_data); singa::Layer* ToLayer(); }; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/layer.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/layer.py b/tool/python/singa/layer.py index 35da9aa..f786245 100644 --- a/tool/python/singa/layer.py +++ b/tool/python/singa/layer.py @@ -101,33 +101,21 @@ class Layer(object): self.singalayer.ComputeFeature(1, self.singaSrclayerVector) - def ComputeGradient(self, step, upd=None): + def ComputeGradient(self): ''' The method creates singa::Updater and calls ComputeGradient for gradient computation then updates the parameters. - - step = (int) // a training step - upd = (object) // Updater object ''' - # create singa::Updater - assert upd != None, 'required Updater (see model.py)' - if Layer.singaupdater == None: - Layer.singaupdater = SingaUpdater.CreateUpdater( - upd.proto.SerializeToString()) - # call ComputeGradient of Singa self.singalayer.ComputeGradient(1, self.singaSrclayerVector) + def UpdateParams(self, step, upd): + ''' The method updates parameter values + ''' # update parameters singaParams = self.singalayer.GetParams() for par in singaParams: - Layer.singaupdater.Update(step, par, 1.0) - - # recursively call ComputeGradient of srclayers - #(TODO) what if there are multiple source layers? - for sly in self.srclayers: - if sly.srclayers != None: - sly.ComputeGradient(step, upd) + upd.singaupdater.Update(step, par, 1.0) def GetParams(self): ''' The method gets parameter values @@ -181,15 +169,13 @@ class Layer(object): class Dummy(object): - def __init__(self, shape=[], path='', dtype='', src=[], **kwargs): + def __init__(self, **kwargs): ''' Dummy layer is used for data layer to feed/fetch input data - shape = (list) // [# of samples, # of channels, img h, img w] - path = (string) // path to dataset + or label information ''' self.is_datalayer = True self.srclayers = None self.singalayer = None - if 'is_label' in kwargs: self.is_label = kwargs['is_label'] # create layer proto for Dummy layer kwargs = {'name':'dummy', 'type':kDummy} @@ -206,155 +192,64 @@ class Dummy(object): self.singalayer.Setup(self.layer.SerializeToString(), layerVector(0)) - def Feed(self, data, nb_channel=1, is_label=0): + def Feed(self, shape, data, aux_data): ''' Create and Setup singa::DummyLayer for input data Insert data using Feed() ''' - batchsize, hdim = data.shape + batchsize = shape[0] + hdim = reduce(lambda x, y: x*y, shape[1:]) datasize = batchsize * hdim # create and setup the dummy layer if self.singalayer == None: - imgsize = int(np.sqrt(hdim/nb_channel)) - shapeVector = [batchsize, nb_channel, imgsize, imgsize] - self.setup(shapeVector) - - data = data.astype(np.float) - dataVector = floatVector(datasize) - k = 0 - for i in range(batchsize): - for j in range(hdim): - dataVector[k] = data[i, j] - k += 1 - self.singalayer.Feed(batchsize, dataVector, is_label) - - def FetchData(self, batchsize): - sidx = self.batch_index * batchsize - eidx = sidx + batchsize - batch = self.data[sidx:eidx, :] - - self.Feed(batch, self.shape[1], self.is_label) - - self.batch_index += 1 - if eidx > self.data.shape[0]: - self.batch_index = 0 + self.setup(shape) + + if data != None: + data = data.astype(np.float) + dataVector = floatVector(datasize) + for i in range(batchsize): + for j in range(hdim): + dataVector[i*hdim+j] = data[i, j] + labelVector = intVector(0) + + if aux_data != None: + aux_data = aux_data.astype(np.int) + labelVector = intVector(datasize) + for i in range(batchsize): + labelVector[i] = aux_data[i, 0] + dataVector = floatVector(0) + + self.singalayer.Feed(batchsize, dataVector, labelVector) def get_singalayer(self): return self.singalayer.ToLayer() -class ImageData(Dummy): - ''' This class will be used for Rafiki, dlaas +class ImageInput(Dummy): + ''' This class is used to feed image data ''' - def __init__(self, shape=[], data_path='', data_type='byte', src=[], - mean_path='', mean_type='float'): - ''' Dummy layer is used for data layer - shape = (list) // [# of samples, # of channels, img h, img w] - data_path = (string) // path to dataset - mean_path - ''' - is_label = False - super(ImageData, self).__init__(shape, data_path, data_type, src, - is_label=is_label, - mean_path=mean_path, - mean_type=mean_type) - - # if dataset path is not specified, skip - # otherwise, load dataset - if data_path == '' or mean_path == '': - return - - self.shape = shape - self.data_path = data_path - self.mean_path = mean_path - self.src = None - self.batch_index = 0 - - nb_samples = shape[0] - nb_pixels = shape[1] - for i in range(len(shape)-2): - nb_pixels *= shape[i+2] - - if data_type == 'byte': - d = np.fromfile(data_path, dtype=np.uint8) - elif data_type == 'int': - d = np.fromfile(data_path, dtype=np.int) - self.data = d.reshape(nb_samples, nb_pixels) - - if mean_type == 'float': - d = np.fromfile(mean_path, dtype=np.float32) - self.mean = d.reshape(1, nb_pixels) - - def Feed(self, data, nb_channel=1, is_label=0): - ''' Create and Setup singa::DummyLayer for input data - Insert data using Feed() - Need to minus the mean file - ''' - batchsize, hdim = data.shape - datasize = batchsize * hdim - - # create and setup the dummy layer - if self.singalayer == None: - imgsize = int(np.sqrt(hdim/nb_channel)) - shapeVector = [batchsize, nb_channel, imgsize, imgsize] - self.setup(shapeVector) - - # feed input data and minus mean - data = data.astype(np.float) - dataVector = floatVector(datasize) - k = 0 - for i in range(batchsize): - for j in range(hdim): - dataVector[k] = data[i, j]-self.mean[0, j] - k += 1 - self.singalayer.Feed(batchsize, dataVector, is_label) - -class LabelData(Dummy): - ''' This class will be used for Rafiki, dlaas + def __init__(self, width=None, height=None, nb_channel=1): + super(ImageInput, self).__init__() + self.width = width + self.height = height + self.nb_channel = nb_channel + + def Feed(self, image_data): + batchsize = image_data.shape[0] + if self.width == None or self.height == None: + hdim = image_data.shape[1] + imgsize = int(np.sqrt(hdim/self.nb_channel)) + shape = [batchsize, self.nb_channel, self.width, self.height] + Dummy.Feed(self, shape, image_data, None) + +class LabelInput(Dummy): + ''' This class is used to feed label data ''' - def __init__(self, shape=[], label_path='', label_type='int', src=[]): - ''' Dummy layer is used for label data layer - shape = (list) // [# of samples, # of channels, img h, img w] - data_path = (string) // path to dataset - mean_path - ''' - is_label = True - super(LabelData, self).__init__(shape, label_path, label_type, src, - is_label=is_label) - - # if dataset path is not specified, skip - # otherwise, load dataset - if label_path == '': - return - - self.shape = shape - self.label_path = label_path - self.src = None - self.batch_index = 0 - - nb_samples = shape[0] - - if label_type == 'int': - d = np.fromfile(label_path, dtype=np.int) - self.data = d.reshape(nb_samples, 1) - - def Feed(self, data, nb_chanel=1, is_label=1): - ''' Create and Setup singa::DummyLayer for input data - Insert data using Feed() - Need to minus the mean file - ''' - batchsize = data.shape[0] - - # create and setup the dummy layer - if self.singalayer == None: - shapeVector = [batchsize, 1] - self.setup(shapeVector) - - data = data.astype(np.float) - dataVector = floatVector(batchsize) - for i in range(batchsize): - dataVector[i] = data[i, 0] - self.singalayer.Feed(batchsize, dataVector, 1) + def __init__(self): + super(LabelInput, self).__init__() + def Feed(self, label_data): + Dummy.Feed(self, label_data.shape, None, label_data) + class Data(Layer): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/model.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py index b6664bd..4a6a688 100644 --- a/tool/python/singa/model.py +++ b/tool/python/singa/model.py @@ -32,6 +32,8 @@ from singa.utils.utility import * from singa.utils.message import * from google.protobuf import text_format +from singa.driver import Updater as SingaUpdater + class Model(object): ''' Configure model parameter - add(): add layer @@ -487,7 +489,21 @@ class Updater(object): setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto) self.proto = upd + self.singaupdater = None + def Update(self, step, layer): + ''' This method updates parameters of layer + step = (int) // training step, i.e., param version + ''' + if self.singaupdater == None: + self.singaupdater = SingaUpdater.CreateUpdater( + self.proto.SerializeToString()) + + # update parameters + singaParams = layer.singalayer.GetParams() + for par in singaParams: + self.singaupdater.Update(step, par, 1.0) + class SGD(Updater): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/utils/utility.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/utils/utility.py b/tool/python/singa/utils/utility.py index bea784d..b88720c 100644 --- a/tool/python/singa/utils/utility.py +++ b/tool/python/singa/utils/utility.py @@ -84,8 +84,3 @@ def setval(proto, **kwargs): else: setattr(proto, key, val) -def swap32(x): - return (((x << 24) & 0xFF000000) | - ((x << 8) & 0x00FF0000) | - ((x >> 8) & 0x0000FF00) | - ((x >> 24) & 0x000000FF))
