SINGA-113 Model/Hybrid Partition Support minor change after rebase to master branch * pass compilation * change Blob A.CopyFrom(B) to Copy(A, B) * make LayerProto.partition_dim stand for own partition dimension only * change InnerProduct layer connection to OneToAll
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8af565cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8af565cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8af565cb Branch: refs/heads/master Commit: 8af565cb7c120ee97b5722b8356c27006c0ebbf7 Parents: 2d38cb3 Author: WANG Sheng <[email protected]> Authored: Wed Dec 16 21:11:56 2015 +0800 Committer: Wei Wang <[email protected]> Committed: Fri Dec 25 20:26:28 2015 +0800 ---------------------------------------------------------------------- include/singa/neuralnet/connection_layer.h | 11 +++++++ include/singa/neuralnet/neuron_layer.h | 3 ++ src/neuralnet/connection_layer/concate.cc | 22 +++++++------- src/neuralnet/connection_layer/slice.cc | 40 +++++++++++++------------ src/neuralnet/connection_layer/split.cc | 7 ++--- src/neuralnet/neuralnet.cc | 17 ++++++++--- src/neuralnet/neuron_layer/dummy.cc | 8 ++--- src/proto/job.proto | 10 ++++++- src/test/test_connection_layers.cc | 16 +++++----- src/test/test_neuralnet.cc | 6 ++-- 10 files changed, 85 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/include/singa/neuralnet/connection_layer.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h index c9b0caf..ae918ce 100644 --- a/include/singa/neuralnet/connection_layer.h +++ b/include/singa/neuralnet/connection_layer.h @@ -83,6 +83,10 @@ class ConcateLayer : public ConnectionLayer { void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; + + private: + int num_concates = 0; + int concate_dim = 0; }; /** @@ -102,6 +106,10 @@ class SliceLayer : public ConnectionLayer { const Blob<float>& grad(const Layer* from) const override; Blob<float>* mutable_data(const Layer* from) override; Blob<float>* mutable_grad(const Layer* from) override; + + private: + int num_slices = 0; + int slice_dim = 0; }; /** @@ -120,6 +128,9 @@ class SplitLayer : public ConnectionLayer { const std::string ToString(bool debug, int flag) override; const Blob<float>& grad(const Layer* from) const override; Blob<float>* mutable_grad(const Layer* from) override; + + private: + int num_splits = 0; }; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/include/singa/neuralnet/neuron_layer.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h index 0b936c4..b02aa7d 100644 --- a/include/singa/neuralnet/neuron_layer.h +++ b/include/singa/neuralnet/neuron_layer.h @@ -141,6 +141,9 @@ class InnerProductLayer : public NeuronLayer { void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; + ConnectionType src_neuron_connection(int k) const override { + return kOneToAll; + } const std::vector<Param*> GetParams() const override { std::vector<Param*> params{weight_, bias_}; return params; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/concate.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc index 13a7ea7..adf701e 100644 --- a/src/neuralnet/connection_layer/concate.cc +++ b/src/neuralnet/connection_layer/concate.cc @@ -28,13 +28,15 @@ void ConcateLayer::Setup(const LayerProto& conf, CHECK_GT(srclayers.size(), 1); Layer::Setup(conf, srclayers); vector<int> shape = srclayers[0]->data(this).shape(); - CHECK_GE(partition_dim(), 0); - CHECK_LT(partition_dim(), shape.size()); - CHECK_EQ(num_partitions(), srclayers.size()); + concate_dim = conf.concate_conf().concate_dim(); + num_concates = conf.concate_conf().num_concates(); + CHECK_GE(concate_dim, 0); + CHECK_LT(concate_dim, shape.size()); + CHECK_EQ(num_concates, srclayers.size()); for (size_t i = 1; i < srclayers.size(); i++) { const vector<int>& src_shape = srclayers[i]->data(this).shape(); for (size_t j = 0; j < shape.size(); j++) - if (static_cast<int>(j) == partition_dim()) + if (static_cast<int>(j) == concate_dim) shape[j] += src_shape[j]; else CHECK_EQ(shape[j], src_shape[j]); @@ -45,10 +47,10 @@ void ConcateLayer::Setup(const LayerProto& conf, void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { CHECK_GT(srclayers.size(), 1); - CHECK_EQ(num_partitions(), srclayers.size()); + CHECK_EQ(num_concates, srclayers.size()); // calculate step for each memcpy - int step = srclayers[0]->data(this).shape()[partition_dim()]; - for (unsigned i = partition_dim() + 1; i < data_.shape().size(); ++i) + int step = srclayers[0]->data(this).shape()[concate_dim]; + for (unsigned i = concate_dim + 1; i < data_.shape().size(); ++i) step *= data_.shape()[i]; int srclayer_offset = 0; int concate_offset = 0; @@ -65,10 +67,10 @@ void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { CHECK_GT(srclayers.size(), 1); - CHECK_EQ(num_partitions(), srclayers.size()); + CHECK_EQ(num_concates, srclayers.size()); // calculate step for each memcpy - int step = srclayers[0]->grad(this).shape()[partition_dim()]; - for (unsigned i = partition_dim() + 1; i < grad_.shape().size(); ++i) + int step = srclayers[0]->grad(this).shape()[concate_dim]; + for (unsigned i = concate_dim + 1; i < grad_.shape().size(); ++i) step *= grad_.shape()[i]; int srclayer_offset = 0; int concate_offset = 0; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/slice.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc index ee635e0..db83f1b 100644 --- a/src/neuralnet/connection_layer/slice.cc +++ b/src/neuralnet/connection_layer/slice.cc @@ -38,19 +38,21 @@ void SliceLayer::Setup(const LayerProto& conf, CHECK_EQ(srclayers.size(), 1); Layer::Setup(conf, srclayers); vector<int> shape = srclayers[0]->data(this).shape(); - CHECK_GE(partition_dim(), 0); - CHECK_LT(partition_dim(), shape.size()); - CHECK_GT(num_partitions(), 0); - // add num_partitions()-1 more blobs - for (int i = 1; i < num_partitions(); ++i) { + slice_dim = conf.slice_conf().slice_dim(); + num_slices = conf.slice_conf().num_slices(); + CHECK_GE(slice_dim, 0); + CHECK_LT(slice_dim, shape.size()); + CHECK_GT(num_slices, 0); + // add num_slices-1 more blobs + for (int i = 1; i < num_slices; ++i) { datavec_.push_back(new Blob<float>()); gradvec_.push_back(new Blob<float>()); } // TODO(wangsh): remove equal-size restrict later - CHECK_EQ(shape[partition_dim()] % num_partitions(), 0); - shape[partition_dim()] /= num_partitions(); - for (int i = 0; i < num_partitions(); ++i) { - // if (i == slice_num - 1) shape[partition_dim()] += remain; + CHECK_EQ(shape[slice_dim] % num_slices, 0); + shape[slice_dim] /= num_slices; + for (int i = 0; i < num_slices; ++i) { + // if (i == slice_num - 1) shape[slice_dim] += remain; datavec_[i]->Reshape(shape); gradvec_[i]->Reshape(shape); } @@ -60,13 +62,13 @@ void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { CHECK_EQ(srclayers.size(), 1); const Blob<float>& blob = srclayers[0]->data(this); // calculate step for each memcpy - int step = datavec_[0]->shape()[partition_dim()]; - for (unsigned i = partition_dim() + 1; i < datavec_[0]->shape().size(); ++i) + int step = datavec_[0]->shape()[slice_dim]; + for (unsigned i = slice_dim + 1; i < datavec_[0]->shape().size(); ++i) step *= datavec_[0]->shape()[i]; int srclayer_offset = 0; int slice_offset = 0; while (srclayer_offset < blob.count()) { - for (int i = 0; i < num_partitions(); ++i) { + for (int i = 0; i < num_slices; ++i) { const float* src = blob.cpu_data() + srclayer_offset; float* dst = datavec_[i]->mutable_cpu_data() + slice_offset; memcpy(dst, src, step * sizeof(float)); @@ -80,13 +82,13 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { CHECK_EQ(srclayers.size(), 1); Blob<float>* blob = srclayers[0]->mutable_grad(this); // calculate step for each memcpy - int step = gradvec_[0]->shape()[partition_dim()]; - for (size_t i = partition_dim() + 1; i < gradvec_[0]->shape().size(); ++i) + int step = gradvec_[0]->shape()[slice_dim]; + for (size_t i = slice_dim + 1; i < gradvec_[0]->shape().size(); ++i) step *= gradvec_[0]->shape()[i]; int srclayer_offset = 0; int slice_offset = 0; while (srclayer_offset < blob->count()) { - for (int i = 0; i < num_partitions(); ++i) { + for (int i = 0; i < num_slices; ++i) { const float* src = gradvec_[i]->cpu_data() + slice_offset; float* dst = blob->mutable_cpu_data() + srclayer_offset; memcpy(dst, src, step * sizeof(float)); @@ -98,26 +100,26 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { const Blob<float>& SliceLayer::data(const Layer* from) const { int id = from ? from->partition_id() : 0; - CHECK_LT(id, num_partitions()); + CHECK_LT(id, num_slices); return *datavec_[id]; } const Blob<float>& SliceLayer::grad(const Layer* from) const { int id = from ? from->partition_id() : 0; - CHECK_LT(id, num_partitions()); + CHECK_LT(id, num_slices); return *gradvec_[id]; } Blob<float>* SliceLayer::mutable_data(const Layer* from) { int id = from ? from->partition_id() : 0; CHECK(from); - CHECK_LT(id, num_partitions()); + CHECK_LT(id, num_slices); return datavec_[id]; } Blob<float>* SliceLayer::mutable_grad(const Layer* from) { int id = from ? from->partition_id() : 0; - CHECK_LT(id, num_partitions()); + CHECK_LT(id, num_slices); return gradvec_[id]; } const std::string SliceLayer::ToString(bool debug, int flag) { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/split.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc index 0ebadc2..9bbe59e 100644 --- a/src/neuralnet/connection_layer/split.cc +++ b/src/neuralnet/connection_layer/split.cc @@ -57,12 +57,9 @@ void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { CHECK_EQ(srclayers.size(), 1); // aggregate all gradients to grad_[0] for (int i = 1; i < num_splits; ++i) - AXPY<float>(cpu, 1.0, *gradvec_[i], gradvec_[0]); -// for (int i = 1; i < num_splits; ++i) -// for (int j = 0; j < gradvec_[0]->count(); ++j) -// gradvec_[0]->mutable_cpu_data()[j] += gradvec_[i]->cpu_data()[j]; + AXPY<float>(1.0, *gradvec_[i], gradvec_[0]); // copy grad_[0] to srclayer's grad - srclayers[0]->mutable_grad(this)->CopyFrom(*gradvec_[0]); + Copy(*gradvec_[0], srclayers[0]->mutable_grad(this)); } const Blob<float>& SplitLayer::grad(const Layer* from) const { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/neuralnet.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc index 238bd3f..dfd477e 100644 --- a/src/neuralnet/neuralnet.cc +++ b/src/neuralnet/neuralnet.cc @@ -142,7 +142,7 @@ void NeuralNet::Load(const vector<string>& paths, } } -void NeuralNet::ShareParamsFrom(NeuralNet* other) { +void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) { for (auto& layer : layers_) { auto otherlayer = other->name2layer(layer->name()); if (otherlayer != nullptr) { @@ -281,8 +281,11 @@ NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto, LayerProto* concate_layer = net_w_connection.add_layer(); concate_layer->set_name(concateName(split_layer->name())); concate_layer->set_type(kConcate); + concate_layer->set_partition_dim(dst_layer->partition_dim()); // concate on src_pdim - concate_layer->set_partition_dim(split_layer->partition_dim()); + concate_layer->mutable_concate_conf() + ->set_concate_dim(src_layer->partition_dim()); + concate_layer->mutable_concate_conf()->set_num_concates(npartitions); concate_layer->add_srclayers(split_layer->name()); // connect dst_layer to concate layer dst_layer->add_srclayers(concate_layer->name()); @@ -296,15 +299,21 @@ NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto, LayerProto* slice_layer = net_w_connection.add_layer(); slice_layer->set_name(sliceName(src_layer->name())); slice_layer->set_type(kSlice); + slice_layer->set_partition_dim(src_layer->partition_dim()); // slice on dst_pdim - slice_layer->set_partition_dim(dst_layer->partition_dim()); + slice_layer->mutable_slice_conf() + ->set_slice_dim(dst_layer->partition_dim()); + slice_layer->mutable_slice_conf()->set_num_slices(npartitions); slice_layer->add_srclayers(src_layer->name()); // add concate layer LayerProto* concate_layer = net_w_connection.add_layer(); concate_layer->set_name(concateName(slice_layer->name())); concate_layer->set_type(kConcate); + concate_layer->set_partition_dim(dst_layer->partition_dim()); // concate on src_pdim - concate_layer->set_partition_dim(src_layer->partition_dim()); + concate_layer->mutable_concate_conf() + ->set_concate_dim(src_layer->partition_dim()); + concate_layer->mutable_concate_conf()->set_num_concates(npartitions); concate_layer->add_srclayers(slice_layer->name()); // connect dst_layer to concate layer dst_layer->add_srclayers(concate_layer->name()); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/neuron_layer/dummy.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc index a242f75..11ce3ec 100644 --- a/src/neuralnet/neuron_layer/dummy.cc +++ b/src/neuralnet/neuron_layer/dummy.cc @@ -21,6 +21,7 @@ #include <glog/logging.h> #include "singa/neuralnet/neuron_layer.h" +#include "singa/utils/math_blob.h" namespace singa { @@ -55,7 +56,7 @@ void DummyLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { data_.mutable_cpu_data()[i] = dis(gen); } if (srclayers.size() > 0) - data_.CopyFrom(srclayers[0]->data(this)); + Copy(srclayers[0]->data(this), &data_); } void DummyLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { @@ -64,9 +65,8 @@ void DummyLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { for (int i = 0; i < data_.count(); ++i) grad_.mutable_cpu_data()[i] = dis(gen); } - if (srclayers.size() > 0) { - srclayers[0]->mutable_grad(this)->CopyFrom(grad_); - } + if (srclayers.size() > 0) + Copy(grad_, srclayers[0]->mutable_grad(this)); } } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/proto/job.proto ---------------------------------------------------------------------- diff --git a/src/proto/job.proto b/src/proto/job.proto index 035f2c1..f6c8c73 100644 --- a/src/proto/job.proto +++ b/src/proto/job.proto @@ -195,6 +195,8 @@ message LayerProto { optional ActivationProto activation_conf = 54; // configuration for argsort layer optional ArgSortProto argsort_conf = 52; + // configuration for concate layer + optional ConcateProto concate_conf = 46; // configuration for convolution layer optional ConvolutionProto convolution_conf = 30; // configuration for dummy layer @@ -367,6 +369,11 @@ message ArgSortProto { optional int32 topk = 1 [default = 1]; } +message ConcateProto { + optional int32 concate_dim = 1 [default = 0]; + optional int32 num_concates = 2 [default = 1]; +} + message ConvolutionProto { // The number of outputs for the layer optional int32 num_filters = 1; @@ -491,7 +498,8 @@ message ReLUProto { } message SliceProto { - required int32 slice_dim = 1; + optional int32 slice_dim = 1 [default = 0]; + optional int32 num_slices = 2 [default = 1]; } message SoftmaxProto { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/test/test_connection_layers.cc ---------------------------------------------------------------------- diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc index eab349a..5517bde 100644 --- a/src/test/test_connection_layers.cc +++ b/src/test/test_connection_layers.cc @@ -175,8 +175,8 @@ TEST(ConnectionLayerTest, DataSliceTest) { src_slice.push_back(static_cast<Layer*>(&in)); LayerProto proto_slice; proto_slice.set_name("slice"); - proto_slice.set_partition_dim(0); - proto_slice.set_num_partitions(K); + proto_slice.mutable_slice_conf()->set_slice_dim(0); + proto_slice.mutable_slice_conf()->set_num_slices(K); SliceLayer slice; slice.Setup(proto_slice, src_slice); ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N / K); @@ -232,8 +232,8 @@ TEST(ConnectionLayerTest, ModelSliceTest) { src_slice.push_back(static_cast<Layer*>(&in)); LayerProto proto_slice; proto_slice.set_name("slice"); - proto_slice.set_partition_dim(1); - proto_slice.set_num_partitions(K); + proto_slice.mutable_slice_conf()->set_slice_dim(1); + proto_slice.mutable_slice_conf()->set_num_slices(K); SliceLayer slice; slice.Setup(proto_slice, src_slice); ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N); @@ -297,8 +297,8 @@ TEST(ConnectionLayerTest, DataConcateTest) { src_concate.push_back(static_cast<Layer*>(&in[i])); LayerProto proto_concate; proto_concate.set_name("concate"); - proto_concate.set_partition_dim(0); - proto_concate.set_num_partitions(K); + proto_concate.mutable_concate_conf()->set_concate_dim(0); + proto_concate.mutable_concate_conf()->set_num_concates(K); ConcateLayer concate; concate.Setup(proto_concate, src_concate); ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N); @@ -355,8 +355,8 @@ TEST(ConnectionLayerTest, ModelConcateTest) { src_concate.push_back(static_cast<Layer*>(&in[i])); LayerProto proto_concate; proto_concate.set_name("concate"); - proto_concate.set_partition_dim(1); - proto_concate.set_num_partitions(K); + proto_concate.mutable_concate_conf()->set_concate_dim(1); + proto_concate.mutable_concate_conf()->set_num_concates(K); ConcateLayer concate; concate.Setup(proto_concate, src_concate); ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/test/test_neuralnet.cc ---------------------------------------------------------------------- diff --git a/src/test/test_neuralnet.cc b/src/test/test_neuralnet.cc index e2b8dd7..198ee0d 100644 --- a/src/test/test_neuralnet.cc +++ b/src/test/test_neuralnet.cc @@ -21,11 +21,9 @@ #include "gtest/gtest.h" #include "singa/driver.h" +#include "singa/neuralnet/connection_layer.h" #include "singa/neuralnet/neuralnet.h" -#include "singa/neuralnet/neuron_layer/dummy.h" -#include "singa/neuralnet/connection_layer/slice.h" -#include "singa/neuralnet/connection_layer/split.h" -#include "singa/neuralnet/connection_layer/concate.h" +#include "singa/neuralnet/neuron_layer.h" using namespace singa;
