Repository: incubator-singa Updated Branches: refs/heads/dev f488070e3 -> a4d9aab52
SINGA-190 - Add prelu layer and flatten layer Add test for PReLU using cudagpu device. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/a4d9aab5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/a4d9aab5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/a4d9aab5 Branch: refs/heads/dev Commit: a4d9aab52db083f18b42f6231dda93e835dba609 Parents: f488070 Author: jixin <[email protected]> Authored: Tue Jun 14 13:27:31 2016 +0800 Committer: jixin <[email protected]> Committed: Tue Jun 14 13:27:31 2016 +0800 ---------------------------------------------------------------------- src/model/layer/prelu.cc | 7 +-- src/model/layer/prelu.h | 1 + test/singa/test_prelu.cc | 109 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/src/model/layer/prelu.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/prelu.cc b/src/model/layer/prelu.cc index b916bed..6766d75 100644 --- a/src/model/layer/prelu.cc +++ b/src/model/layer/prelu.cc @@ -41,7 +41,7 @@ const Tensor PReLU::Forward(int flag, const Tensor &input) { h = temp.shape(2); w = temp.shape(3); temp.Reshape(Shape{n * c, h * w}); - Tensor temp_a(Shape{n, c}); + Tensor temp_a(Shape{n, c}, input.device(), input.data_type()); Uniform(1.f, 1.f, &temp_a); MultRow(a_, &temp_a); temp_a.Reshape(Shape{n * c}); @@ -87,7 +87,7 @@ const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag, h = temp1.shape(2); w = temp1.shape(3); temp1.Reshape(Shape{n * c, h * w}); - Tensor temp_a(Shape{n, c}); + Tensor temp_a(Shape{n, c}, grad.device(), grad.data_type()); Uniform(1.f, 1.f, &temp_a); MultRow(a_, &temp_a); temp_a.Reshape(Shape{n * c}); @@ -108,8 +108,9 @@ const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag, LOG(FATAL) << "Incorrect input format for prelu layer."; } input_grad = grad * input * ((input > 0.f) + temp1); - Tensor temp2 = grad * input * (input <= 0.f), temp3(Shape{n * c}); + Tensor temp2 = grad * input * (input <= 0.f); if (format_ == "NCHW") { + Tensor temp3(Shape{n * c}, grad.device(), grad.data_type()); temp2.Reshape(Shape{n * c, h * w}); SumColumns(temp2, &temp3); temp3.Reshape(Shape{n, c}); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/src/model/layer/prelu.h ---------------------------------------------------------------------- diff --git a/src/model/layer/prelu.h b/src/model/layer/prelu.h index d165fe2..2ee5e9f 100644 --- a/src/model/layer/prelu.h +++ b/src/model/layer/prelu.h @@ -21,6 +21,7 @@ #include <string> #include <vector> #include "singa/model/layer.h" +#include "singa_config.h" namespace singa { class PReLU : public Layer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/test/singa/test_prelu.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_prelu.cc b/test/singa/test_prelu.cc index 6fc372b..883935d 100644 --- a/test/singa/test_prelu.cc +++ b/test/singa/test_prelu.cc @@ -135,3 +135,112 @@ TEST(PReLU, BackwardCPU) { for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(dx[i], xptr[i]); for (size_t i = 0; i < params; i++) EXPECT_FLOAT_EQ(da[i], aptr[i]); } + +#ifdef USE_CUDA +TEST(PReLU, ForwardGPU) { + const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f, + -1.f, 2.f, -1.f, -2.f, -2.f, -1.f}; + size_t n = sizeof(x) / sizeof(float); + size_t batchsize = 2, c = 3, h = 2, w = 1; + singa::CudaGPU cuda(0, 1); + singa::Tensor in(singa::Shape{batchsize, h, w, c}, &cuda); + in.CopyDataFromHostPtr<float>(x, n); + + PReLU prelu; + singa::LayerConf conf; + singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); + preluconf->set_channel_shared(false); + preluconf->set_format("NHWC"); + prelu.Setup(conf); + + const float neg_slope[] = {0.25f, 0.5f, 0.75f}; + singa::Tensor a(singa::Shape{c}, &cuda); + a.CopyDataFromHostPtr<float>(neg_slope, c); + prelu.Set_a(a); + + singa::Tensor out = prelu.Forward(singa::kTrain, in); + singa::CppCPU host(0, 1); + out.ToDevice(&host); + const float *yptr = out.data<const float *>(); + EXPECT_EQ(n, out.Size()); + + float *y = new float[n]; + size_t div_factor = prelu.Channel_shared() ? c : 1; + if (prelu.Format() == "NCHW") { + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f); + } + } else if (prelu.Format() == "NHWC") { + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f); + } + } + for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(y[i], yptr[i]); +} + +TEST(PReLU, BackwardGPU) { + const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f, + -1.f, 2.f, -1.f, -2.f, -2.f, -1.f}; + size_t n = sizeof(x) / sizeof(float); + size_t batchsize = 2, c = 3, h = 2, w = 1; + singa::CudaGPU cuda(0, 1); + singa::Tensor in(singa::Shape{batchsize, c, h, w}, &cuda); + in.CopyDataFromHostPtr<float>(x, n); + + PReLU prelu; + singa::LayerConf conf; + singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); + preluconf->set_channel_shared(false); + preluconf->set_format("NCHW"); + prelu.Setup(conf); + + const float neg_slope[] = {0.25f, 0.5f, 0.75f}; + singa::Tensor a(singa::Shape{c}, &cuda); + a.CopyDataFromHostPtr<float>(neg_slope, c); + prelu.Set_a(a); + + singa::Tensor out = prelu.Forward(singa::kTrain, in); + const float grad[] = {1.f, 2.f, -2.f, -1.f, -1.f, -3.f, + 2.f, -2.f, 1.f, 1.f, -2.f, 0.f}; + singa::Tensor out_diff(singa::Shape{batchsize, c, h, w}, &cuda); + out_diff.CopyDataFromHostPtr<float>(grad, n); + const auto ret = prelu.Backward(singa::kTrain, out_diff); + + singa::Tensor in_diff = ret.first; + singa::CppCPU host(0, 1); + in_diff.ToDevice(&host); + const float *xptr = in_diff.data<const float *>(); + singa::Tensor a_diff = ret.second.at(0); + a_diff.ToDevice(&host); + const float *aptr = a_diff.data<const float *>(); + float *dx = new float[n]; + size_t div_factor = prelu.Channel_shared() ? c : 1; + size_t params = prelu.Channel_shared() ? 1 : c; + float da[] = {0.f, 0.f, 0.f}; + if (prelu.Format() == "NCHW") { + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + dx[i] = grad[i] * + (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f)); + } + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + da[pos] += grad[i] * std::min(x[i], 0.f); + } + } else if (prelu.Format() == "NHWC") { + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + dx[i] = grad[i] * + (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f)); + } + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + da[pos] += grad[i] * std::min(x[i], 0.f); + } + } + for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(dx[i], xptr[i]); + for (size_t i = 0; i < params; i++) EXPECT_FLOAT_EQ(da[i], aptr[i]); +} +#endif
