Repository: incubator-singa Updated Branches: refs/heads/tutorial [created] 7306bcffb
SINGA-65 Add an example of implementing user's own layers The folder examples/mlp includes files for a user-defined layer, i.e., HiddenLayer class. The main.cc register this class. The myproto.proto defines the configuration of this layer and extends it to the base LayerProto. Makefile.example contains instructions for compiling and linking user code. job.conf specifies a simple model with only one hidden layer. deep.conf configures a deep model with 6 hidden layers, which is slower per iteration, but converges faster than the simple model. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/98b49952 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/98b49952 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/98b49952 Branch: refs/heads/tutorial Commit: 98b499527012ab6b61fd854b39225b35d7370716 Parents: 50deedd Author: wangwei <[email protected]> Authored: Thu Sep 3 09:57:03 2015 +0800 Committer: wangwei <[email protected]> Committed: Fri Sep 4 20:24:45 2015 +0800 ---------------------------------------------------------------------- examples/cifar10/job.conf | 6 +- examples/mlp/Makefile.example | 8 ++ examples/mlp/deep.conf | 221 +++++++++++++++++++++++++++++++++++++ examples/mlp/hidden_layer.cc | 72 ++++++++++++ examples/mlp/hidden_layer.h | 22 ++++ examples/mlp/job.conf | 95 ++++++++++++++++ examples/mlp/main.cc | 25 +++++ examples/mlp/myproto.proto | 11 ++ src/utils/tool.cc | 43 +++++++- 9 files changed, 498 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/cifar10/job.conf ---------------------------------------------------------------------- diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf index 0fdd244..90705e0 100644 --- a/examples/cifar10/job.conf +++ b/examples/cifar10/job.conf @@ -3,10 +3,12 @@ train_steps: 1000 test_steps: 100 test_freq:300 disp_freq:30 -train_one_batch { +train_one_batch + +{ alg: kBP } -updater{ + updater { type: kSGD weight_decay:0.004 learning_rate { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/mlp/Makefile.example b/examples/mlp/Makefile.example new file mode 100644 index 0000000..71edd71 --- /dev/null +++ b/examples/mlp/Makefile.example @@ -0,0 +1,8 @@ +# refer to examples/mnist/ for data creation +MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 + +all: + protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. myproto.proto + $(CXX) main.cc hidden_layer.cc myproto.pb.cc $(MSHADOW_FLAGS) -std=c++11 -lsinga -lglog -lprotobuf -lopenblas -I../../include\ + -I../../include/proto/ -L../../.libs/ -L/usr/local -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/\ + -o mlp.bin http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/deep.conf ---------------------------------------------------------------------- diff --git a/examples/mlp/deep.conf b/examples/mlp/deep.conf new file mode 100644 index 0000000..588fefa --- /dev/null +++ b/examples/mlp/deep.conf @@ -0,0 +1,221 @@ +name: "mlp" +train_steps: 1200 +test_steps:10 +test_freq:60 +disp_freq:10 +train_one_batch { + alg: kBP +} +updater{ + type: kSGD + learning_rate{ + type : kStep + base_lr: 0.001 + step_conf{ + change_freq: 60 + gamma: 0.997 + } + } +} +neuralnet { + layer { + name: "data" + type: kShardData + sharddata_conf { + path: "examples/mnist/mnist_train_shard" + batchsize: 1000 + } + exclude: kTest + } + + layer { + name: "data" + type: kShardData + sharddata_conf { + path: "examples/mnist/mnist_test_shard" + batchsize: 1000 + } + exclude: kTrain + } + + layer{ + name:"mnist" + type: kMnist + srclayers: "data" + mnist_conf { + norm_a: 127.5 + norm_b: 1 + } + } + layer{ + name: "label" + type: kLabel + srclayers: "data" + } + + layer{ + name: "hid1" + user_type: "kHidden" + srclayers:"mnist" + [singa.hidden_conf] { + num_output: 2500 + } + param{ + name: "w1" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b1" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + + layer{ + name: "hid2" + user_type: "kHidden" + srclayers:"hid1" + [singa.hidden_conf] { + num_output: 2000 + } + param{ + name: "w2" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b2" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + + layer{ + name: "hid3" + user_type: "kHidden" + srclayers:"hid2" + [singa.hidden_conf] { + num_output: 1500 + } + param{ + name: "w3" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b3" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + + layer{ + name: "hid4" + user_type: "kHidden" + srclayers:"hid3" + [singa.hidden_conf] { + num_output: 1000 + } + param{ + name: "w4" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b4" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + + + layer{ + name: "hid5" + user_type: "kHidden" + srclayers:"hid4" + [singa.hidden_conf] { + num_output: 500 + } + param{ + name: "w5" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b5" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + + + layer{ + name: "hid6" + user_type: "kHidden" + srclayers:"hid5" + [singa.hidden_conf] { + num_output: 10 + } + param{ + name: "w6" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b6" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + layer{ + name: "loss" + type:kSoftmaxLoss + softmaxloss_conf{ + topk:1 + } + srclayers:"hid6" + srclayers:"label" + } +} +cluster { + nworker_groups: 1 + nserver_groups: 1 + workspace: "examples/mnist" +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/hidden_layer.cc ---------------------------------------------------------------------- diff --git a/examples/mlp/hidden_layer.cc b/examples/mlp/hidden_layer.cc new file mode 100644 index 0000000..f4dacf2 --- /dev/null +++ b/examples/mlp/hidden_layer.cc @@ -0,0 +1,72 @@ +#include "hidden_layer.h" + +#include "mshadow/tensor.h" +#include "mshadow/cxxnet_op.h" + +namespace singa { +using namespace mshadow; +using mshadow::cpu; +using mshadow::Shape1; +using mshadow::Shape2; + +inline Tensor<cpu, 2> NewTensor2(Blob<float>* blob) { + const vector<int>& shape = blob->shape(); + Tensor<cpu, 2> tensor(blob->mutable_cpu_data(), + Shape2(shape[0], blob->count() / shape[0])); + return tensor; +} + +inline Tensor<cpu, 1> NewTensor1(Blob<float>* blob) { + Tensor<cpu, 1> tensor(blob->mutable_cpu_data(), Shape1(blob->count())); + return tensor; +} + + +HiddenLayer::~HiddenLayer() { + delete weight_; + delete bias_; +} + +void HiddenLayer::Setup(const LayerProto& proto, int npartitions) { + Layer::Setup(proto, npartitions); + CHECK_EQ(srclayers_.size(), 1); + const auto& src = srclayers_[0]->data(this); + batchsize_ = src.shape()[0]; + vdim_ = src.count() / batchsize_; + hdim_ = layer_proto_.GetExtension(hidden_conf).num_output(); + data_.Reshape(vector<int>{batchsize_, hdim_}); + grad_.ReshapeLike(data_); + weight_ = Param::Create(proto.param(0)); + bias_ = Param::Create(proto.param(1)); + weight_->Setup(vector<int>{hdim_, vdim_}); + bias_->Setup(vector<int>{hdim_}); +} + +void HiddenLayer::ComputeFeature(int flag, Metric* perf) { + auto data = NewTensor2(&data_); + auto src = NewTensor2(srclayers_[0]->mutable_data(this)); + auto weight = NewTensor2(weight_->mutable_data()); + auto bias = NewTensor1(bias_->mutable_data()); + data = dot(src, weight.T()); + // repmat: repeat bias vector into batchsize rows + data += expr::repmat(bias, batchsize_); + data = expr::F<op::stanh>(data); +} + +void HiddenLayer::ComputeGradient(int flag, Metric* perf) { + auto data = NewTensor2(&data_); + auto src = NewTensor2(srclayers_[0]->mutable_data(this)); + auto grad = NewTensor2(&grad_); + auto weight = NewTensor2(weight_->mutable_data()); + auto gweight = NewTensor2(weight_->mutable_grad()); + auto gbias = NewTensor1(bias_->mutable_grad()); + + grad = expr::F<op::stanh_grad>(data) * grad; + gbias = expr::sum_rows(grad); + gweight = dot(grad.T(), src); + if (srclayers_[0]->mutable_grad(this) != nullptr) { + auto gsrc = NewTensor2(srclayers_[0]->mutable_grad(this)); + gsrc = dot(grad, weight); + } +} +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/hidden_layer.h ---------------------------------------------------------------------- diff --git a/examples/mlp/hidden_layer.h b/examples/mlp/hidden_layer.h new file mode 100644 index 0000000..c305261 --- /dev/null +++ b/examples/mlp/hidden_layer.h @@ -0,0 +1,22 @@ +#include "neuralnet/layer.h" +#include "myproto.pb.h" + +namespace singa { +class HiddenLayer : public NeuronLayer { + public: + ~HiddenLayer(); + void Setup(const LayerProto& proto, int npartitions) override; + void ComputeFeature(int flag, Metric* perf) override; + void ComputeGradient(int flag, Metric* perf) override; + const std::vector<Param*> GetParams() const override { + std::vector<Param*> params{weight_, bias_}; + return params; + } + + private: + int batchsize_; + int vdim_, hdim_; + bool transpose_; + Param *weight_, *bias_; +}; +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/job.conf ---------------------------------------------------------------------- diff --git a/examples/mlp/job.conf b/examples/mlp/job.conf new file mode 100644 index 0000000..163bc19 --- /dev/null +++ b/examples/mlp/job.conf @@ -0,0 +1,95 @@ +name: "mlp" +train_steps: 1200 +test_steps:10 +test_freq:60 +disp_freq:10 +train_one_batch { + alg: kBP +} +updater{ + type: kSGD + learning_rate{ + type : kStep + base_lr: 0.001 + step_conf{ + change_freq: 60 + gamma: 0.997 + } + } +} +neuralnet { + layer { + name: "data" + type: kShardData + sharddata_conf { + path: "examples/mnist/mnist_train_shard" + batchsize: 1000 + } + exclude: kTest + } + + layer { + name: "data" + type: kShardData + sharddata_conf { + path: "examples/mnist/mnist_test_shard" + batchsize: 1000 + } + exclude: kTrain + } + + layer{ + name:"mnist" + type: kMnist + srclayers: "data" + mnist_conf { + norm_a: 127.5 + norm_b: 1 + } + } + layer{ + name: "label" + type: kLabel + srclayers: "data" + } + + + layer{ + name: "hid1" + user_type: "kHidden" + srclayers:"mnist" + [singa.hidden_conf] { + num_output: 10 + } + param{ + name: "w1" + init { + type: kUniform + low:-0.05 + high:0.05 + } + } + param{ + name: "b1" + init { + type : kUniform + low: -0.05 + high:0.05 + } + } + } + layer{ + name: "loss" + type:kSoftmaxLoss + softmaxloss_conf{ + topk:1 + } + srclayers:"hid1" + srclayers:"label" + } +} +cluster { + nworker_groups: 1 + nserver_groups: 1 + workspace: "examples/mnist" +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/main.cc ---------------------------------------------------------------------- diff --git a/examples/mlp/main.cc b/examples/mlp/main.cc new file mode 100644 index 0000000..c27f38d --- /dev/null +++ b/examples/mlp/main.cc @@ -0,0 +1,25 @@ +#include <string> +#include "singa.h" +#include "hidden_layer.h" +#include "myproto.pb.h" +#include "utils/common.h" + +int main(int argc, char **argv) { + // must create driver at the beginning and call its Init method. + singa::Driver driver; + driver.Init(argc, argv); + + // if -resume in argument list, set resume to true; otherwise false + int resume_pos = singa::ArgPos(argc, argv, "-resume"); + bool resume = (resume_pos != -1); + + // users can register new subclasses of layer, updater, etc. + driver.RegisterLayer<singa::HiddenLayer, std::string>("kHidden"); + + // get the job conf, and custmize it if need + singa::JobProto jobConf = driver.job_conf(); + + // submit the job + driver.Submit(resume, jobConf); + return 0; +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/myproto.proto ---------------------------------------------------------------------- diff --git a/examples/mlp/myproto.proto b/examples/mlp/myproto.proto new file mode 100644 index 0000000..ba62ac2 --- /dev/null +++ b/examples/mlp/myproto.proto @@ -0,0 +1,11 @@ +package singa; +import "job.proto"; + +message HiddenProto { + required int32 num_output = 1; + optional bool use_stanh = 2 [default = true]; +} + +extend LayerProto { + optional HiddenProto hidden_conf = 102; +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/src/utils/tool.cc ---------------------------------------------------------------------- diff --git a/src/utils/tool.cc b/src/utils/tool.cc index 084ebea..ea28a6c 100644 --- a/src/utils/tool.cc +++ b/src/utils/tool.cc @@ -2,6 +2,7 @@ #include <algorithm> #include <fstream> #include <iostream> +#include <google/protobuf/text_format.h> #include "proto/job.pb.h" #include "proto/singa.pb.h" #include "utils/cluster_rt.h" @@ -30,12 +31,48 @@ int create() { return SUCCESS; } +// extract cluster configuration part from the job config file +// TODO improve this function to make it robust +const std::string extract_cluster(const char* jobfile) { + std::ifstream fin; + fin.open(jobfile, std::ifstream::in); + CHECK(fin.is_open()) << "cannot open job conf file " << jobfile; + std::string line; + std::string cluster; + while (std::getline(fin, line)) { + // end of extraction (cluster config has not nested messages) + if (line.find("}") != std::string::npos && cluster.length()) { + cluster += line.substr(0, line.find("}")); + break; + } + unsigned int pos = 0; + while (pos < line.length() && line.at(pos) == ' ' ) pos++; + if (line.find("cluster", pos) == pos) { // start with <whitespace> cluster + pos += 7; + do { // looking for the first '{', which may be in the next lines + while (pos < line.length() && + (line.at(pos) == ' ' || line.at(pos) =='\t')) pos++; + if (pos < line.length()) { + CHECK_EQ(line.at(pos), '{') << "error around 'cluster' field"; + cluster = " "; // start extraction + break; + } else + pos = 0; + }while(std::getline(fin, line)); + } else if (cluster.length()) { + cluster += line + "\n"; + } + } + return cluster; +} + + // generate a host list int genhost(char* job_conf) { // compute required #process from job conf - singa::JobProto job; - singa::ReadProtoFromTextFile(job_conf, &job); - singa::ClusterProto cluster = job.cluster(); + singa::ClusterProto cluster; + google::protobuf::TextFormat::ParseFromString(extract_cluster(job_conf), + &cluster); int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group() / cluster.nworkers_per_procs(); int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
