SINGA-111 Add slice, concate and split layers Add concate layer implementation and test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3ea1eb69 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3ea1eb69 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3ea1eb69 Branch: refs/heads/master Commit: 3ea1eb691708b39b4a786f6494f56f6dbfe8e1df Parents: 28e48a6 Author: WANG Sheng <[email protected]> Authored: Tue Dec 8 21:28:30 2015 +0800 Committer: WANG Sheng <[email protected]> Committed: Thu Dec 10 16:22:59 2015 +0800 ---------------------------------------------------------------------- .../singa/neuralnet/connection_layer/concate.h | 3 + src/neuralnet/connection_layer/concate.cc | 52 ++++++-- src/test/test_connection_layers.cc | 119 +++++++++++++++++++ 3 files changed, 164 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/include/singa/neuralnet/connection_layer/concate.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h index 5875835..6e40040 100644 --- a/include/singa/neuralnet/connection_layer/concate.h +++ b/include/singa/neuralnet/connection_layer/concate.h @@ -37,6 +37,9 @@ class ConcateLayer : public ConnectionLayer { void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; + + private: + int concate_dim_; }; } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/neuralnet/connection_layer/concate.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc index fd2fb24..8f519d2 100644 --- a/src/neuralnet/connection_layer/concate.cc +++ b/src/neuralnet/connection_layer/concate.cc @@ -26,30 +26,62 @@ namespace singa { using std::vector; void ConcateLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - Layer::Setup(conf, srclayers); - size_t concate_dim = conf.concate_conf().concate_dim(); - CHECK_GE(concate_dim, 0); + const vector<Layer*>& srclayers) { CHECK_GT(srclayers.size(), 1); + Layer::Setup(conf, srclayers); + concate_dim_ = conf.concate_conf().concate_dim(); vector<int> shape = srclayers[0]->data(this).shape(); + CHECK_GE(concate_dim_, 0); + CHECK_LT(concate_dim_, shape.size()); for (size_t i = 1; i < srclayers.size(); i++) { - const vector<int>& srcshape = srclayers[i]->data(this).shape(); + const vector<int>& src_shape = srclayers[i]->data(this).shape(); for (size_t j = 0; j < shape.size(); j++) - if (j == concate_dim) - shape[j] += srcshape[j]; + if (static_cast<int>(j) == concate_dim_) + shape[j] += src_shape[j]; else - CHECK_EQ(shape[j], srcshape[j]); + CHECK_EQ(shape[j], src_shape[j]); } data_.Reshape(shape); grad_.Reshape(shape); } void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - LOG(FATAL) << "Not implemented for Concate Layer"; + CHECK_GT(srclayers.size(), 1); + // calculate step for each memcpy + int step = srclayers[0]->data(this).shape()[concate_dim_]; + for (unsigned i = concate_dim_ + 1; i < data_.shape().size(); ++i) + step *= data_.shape()[i]; + int srclayer_offset = 0; + int concate_offset = 0; + while (concate_offset < data_.count()) { + for (size_t i = 0; i < srclayers.size(); ++i) { + const float* src = srclayers[i]->data(this).cpu_data() + srclayer_offset; + float* dst = data_.mutable_cpu_data() + concate_offset; + memcpy(dst, src, step * sizeof(float)); + concate_offset += step; + } + srclayer_offset += step; + } } void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - LOG(FATAL) << "Not implemented for Concate Layer"; + CHECK_GT(srclayers.size(), 1); + // calculate step for each memcpy + int step = srclayers[0]->grad(this).shape()[concate_dim_]; + for (unsigned i = concate_dim_ + 1; i < grad_.shape().size(); ++i) + step *= grad_.shape()[i]; + int srclayer_offset = 0; + int concate_offset = 0; + while (concate_offset < grad_.count()) { + for (size_t i = 0; i < srclayers.size(); ++i) { + const float* src = grad_.cpu_data() + concate_offset; + float* dst = srclayers[i]->mutable_grad(this)->mutable_cpu_data() + + srclayer_offset; + memcpy(dst, src, step * sizeof(float)); + concate_offset += step; + } + srclayer_offset += step; + } } } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/test/test_connection_layers.cc ---------------------------------------------------------------------- diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc index a2cb1d3..e072114 100644 --- a/src/test/test_connection_layers.cc +++ b/src/test/test_connection_layers.cc @@ -26,6 +26,7 @@ #include "singa/comm/msg.h" #include "singa/comm/socket.h" #include "singa/neuralnet/connection_layer/bridge.h" +#include "singa/neuralnet/connection_layer/concate.h" #include "singa/neuralnet/connection_layer/slice.h" #include "singa/neuralnet/neuron_layer/dummy.h" #include "singa/proto/job.pb.h" @@ -277,3 +278,121 @@ TEST(ConnectionLayerTest, ModelSliceTest) { out[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step]); } } + +TEST(ConnectionLayerTest, DataConcateTest) { + // use dummy as input layers + LayerProto proto_in[K]; + vector<Layer*> src_in[K]; + DummyLayer in[K]; + for (int i = 0; i < K; ++i) { + proto_in[i].set_name("dummy_input_"+std::to_string(i)); + proto_in[i].set_partition_id(i); + proto_in[i].mutable_dummy_conf()->set_input(true); + proto_in[i].mutable_dummy_conf()->add_shape(N / K); + proto_in[i].mutable_dummy_conf()->add_shape(M); + in[i].Setup(proto_in[i], src_in[i]); + } + + // add concate layer + vector<Layer*> src_concate; + for (int i = 0; i < K; ++i) + src_concate.push_back(static_cast<Layer*>(&in[i])); + LayerProto proto_concate; + proto_concate.set_name("concate"); + proto_concate.mutable_concate_conf()->set_concate_dim(0); + ConcateLayer concate; + concate.Setup(proto_concate, src_concate); + ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N); + ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M); + + // use dummy as output layer + vector<Layer*> src_out; + src_out.push_back(static_cast<Layer*>(&concate)); + LayerProto proto_out; + proto_out.set_name("dummy_output"); + proto_out.mutable_dummy_conf()->set_output(true); + DummyLayer out; + out.Setup(proto_out, src_out); + + // test for computing feature + for (int i = 0; i < K; ++i) + in[i].ComputeFeature(0, src_in[i]); + concate.ComputeFeature(0, src_concate); + out.ComputeFeature(0, src_out); + int step = (N * M) / K; + for (int i = 0; i < out.data(nullptr).count(); ++i) { + ASSERT_EQ(in[i / step].data(nullptr).cpu_data()[i % step], + out.data(nullptr).cpu_data()[i]); + } + + // test for computing gradient + out.ComputeGradient(0, src_out); + concate.ComputeGradient(0, src_concate); + for (int i = 0; i < K; ++i) + in[i].ComputeGradient(0, src_in[i]); + for (int i = 0; i < out.grad(nullptr).count(); ++i) { + ASSERT_EQ(in[i / step].grad(nullptr).cpu_data()[i % step], + out.grad(nullptr).cpu_data()[i]); + } +} + +TEST(ConnectionLayerTest, ModelConcateTest) { + // use dummy as input layers + LayerProto proto_in[K]; + vector<Layer*> src_in[K]; + DummyLayer in[K]; + for (int i = 0; i < K; ++i) { + proto_in[i].set_name("dummy_input_"+std::to_string(i)); + proto_in[i].set_partition_id(i); + proto_in[i].mutable_dummy_conf()->set_input(true); + proto_in[i].mutable_dummy_conf()->add_shape(N); + proto_in[i].mutable_dummy_conf()->add_shape(M / K); + in[i].Setup(proto_in[i], src_in[i]); + } + + // add concate layer + vector<Layer*> src_concate; + for (int i = 0; i < K; ++i) + src_concate.push_back(static_cast<Layer*>(&in[i])); + LayerProto proto_concate; + proto_concate.set_name("concate"); + proto_concate.mutable_concate_conf()->set_concate_dim(1); + ConcateLayer concate; + concate.Setup(proto_concate, src_concate); + ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N); + ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M); + + // use dummy as output layer + vector<Layer*> src_out; + src_out.push_back(static_cast<Layer*>(&concate)); + LayerProto proto_out; + proto_out.set_name("dummy_output"); + proto_out.mutable_dummy_conf()->set_output(true); + DummyLayer out; + out.Setup(proto_out, src_out); + + // test for computing feature + for (int i = 0; i < K; ++i) + in[i].ComputeFeature(0, src_in[i]); + concate.ComputeFeature(0, src_concate); + out.ComputeFeature(0, src_out); + int step = M / K; + int offset = 0; + for (int i = 0; i < out.grad(nullptr).count(); ++i) { + if (i && i % M == 0) offset += step; + ASSERT_EQ(in[(i / step) % K].data(nullptr).cpu_data()[offset + i % step], + out.data(nullptr).cpu_data()[i]); + } + + // test for computing gradient + out.ComputeGradient(0, src_out); + concate.ComputeGradient(0, src_concate); + for (int i = 0; i < K; ++i) + in[i].ComputeGradient(0, src_in[i]); + offset = 0; + for (int i = 0; i < out.grad(nullptr).count(); ++i) { + if (i && i % M == 0) offset += step; + ASSERT_EQ(in[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step], + out.grad(nullptr).cpu_data()[i]); + } +}
