Repository: incubator-singa Updated Branches: refs/heads/master f35d217c9 -> 5afd81c84
SINGA-271 Add Concat and Slice layers Add concat and slice layers Pass unit tests for Slice and Concat layers; Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/16f3bf64 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/16f3bf64 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/16f3bf64 Branch: refs/heads/master Commit: 16f3bf649b9c1fc474304df0d89a515c57b0abac Parents: f35d217 Author: wangwei <[email protected]> Authored: Sun Nov 20 00:10:05 2016 +0800 Committer: wang wei <[email protected]> Committed: Tue Nov 22 06:25:00 2016 +0000 ---------------------------------------------------------------------- include/singa/core/tensor.h | 11 ++- src/core/tensor/tensor.cc | 36 ++++--- src/model/layer/concat.cc | 70 ++++++++++++++ src/model/layer/concat.h | 54 +++++++++++ src/model/layer/cudnn_rnn.cc | 20 ++++ src/model/layer/slice.cc | 72 ++++++++++++++ src/model/layer/slice.h | 54 +++++++++++ src/model/layer/split.cc | 3 + test/singa/test_concat.cc | 193 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 490 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/include/singa/core/tensor.h ---------------------------------------------------------------------- diff --git a/include/singa/core/tensor.h b/include/singa/core/tensor.h index a41afbc..a39217b 100644 --- a/include/singa/core/tensor.h +++ b/include/singa/core/tensor.h @@ -448,21 +448,26 @@ void ComputeCrossEntropy(const Tensor &p, const Tensor &t, Tensor *loss); /// or 2-d matrix. 'grad' has the same shape as 'p'. dx is computed into p. void SoftmaxCrossEntropyBwd(const Tensor &t, Tensor *p); -/// Return a tensor consisting of rows ([start, end)) from 'in'. It shares the -/// memory with 'in'. 'in' is a 1D or 2D Tensor. -Tensor SliceRows(const Tensor &in, const size_t start, const size_t end); /// Return a tensor consisting of rows ([start, end)) from 'in'. It copies the /// values from 'in'. 'in' ia a 2D Tensor. Tensor CopyRows(const Tensor &in, const size_t start, const size_t end); +/// Alias of CopyRows +Tensor SliceRows(const Tensor &in, const size_t start, const size_t end); /// Return a tensor consisting of columns ([start, end)) from 'in'. It copies /// the values from 'in'. 'in' is a 2D Tensor. Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end); +/// Alias of CopyColumns +Tensor SliceColumns(const Tensor &in, const size_t start, const size_t end); /// Return a tensor which is vertically stacked from tensors in 'in'. Each /// tensor in 'in' is a 2D tensor. Values are copied, no memory sharing. Tensor ConcatenateRows(const vector<Tensor> &in); +/// Alias name for function ConcatenateRows +Tensor ConcatRows(const vector<Tensor> &in); /// Return a tensor which is horizontally stacked from tensors in 'in'. Each /// tensor in 'in' is a 2D tensor. Values are copied, no memory sharing. Tensor ConcatenateColumns(const vector<Tensor> &in); +/// Alias name for function ConcatenateColumns +Tensor ConcatColumns(const vector<Tensor> &in); } // namespace singa #endif // SINGA_CORE_TENSOR_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/core/tensor/tensor.cc ---------------------------------------------------------------------- diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc index 424edb2..83e1a00 100644 --- a/src/core/tensor/tensor.cc +++ b/src/core/tensor/tensor.cc @@ -762,7 +762,9 @@ Tensor ConcatenateRows(const vector<Tensor> &in) { } return out; } - +Tensor ConcatRows(const vector<Tensor> &in) { + return ConcatenateRows(in); +} // TODO(wangwei) add a copypatch function for improve the efficiency on GPU. Tensor ConcatenateColumns(const vector<Tensor> &in) { size_t nrow = 0, ncol = 0; @@ -788,10 +790,13 @@ Tensor ConcatenateColumns(const vector<Tensor> &in) { } return out; } +Tensor ConcatColumns(const vector<Tensor> &in) { + return ConcatenateColumns(in); +} Tensor CopyRows(const Tensor &in, const size_t start, const size_t end) { CHECK_LT(start, end); - CHECK_GE(in.shape(0), end); + CHECK_GE(in.shape(0), end) << "Tensor size must >= end"; Shape s = in.shape(); s[0] = end - start; size_t sample_size = in.Size() / in.shape(0); @@ -800,6 +805,10 @@ Tensor CopyRows(const Tensor &in, const size_t start, const size_t end) { return out; } +Tensor SliceRows(const Tensor &in, const size_t start, const size_t end) { + return CopyRows(in, start, end); +} + Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end) { CHECK_EQ(in.nDim(), 2u); CHECK_LT(start, end); @@ -814,6 +823,11 @@ Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end) { return out; } +Tensor SliceColumns(const Tensor &in, const size_t start, const size_t end) { + return CopyColumns(in, start, end); +} + + /// Divide row 'v' by each row of matrix M; write results into 'out' void DivRow(const Tensor &v, Tensor *M) { Tensor inv; @@ -851,24 +865,6 @@ void MultRow(const Tensor &v, Tensor *M) { }); } -Tensor SliceRows(const Tensor &in, const size_t start, const size_t end) { - LOG(FATAL) << "Tensor::SliceRows is not implemented"; - Tensor ret; - /* - CHECK_LE(in.nDim(), 2); - CHECK_LT(start, end); - CHECK_LE(in.shape(0), end); - Shape s; - if (in.nDim() == 2) - s = Shape{end - start, in.shape(1)}; - else - s = Shape{end - start}; - Tensor out(s, in.device(), in.data_type()); - Block *b = out.block(); - */ - return ret; -} - void SubColumn(const Tensor &v, Tensor *M) { AddColumn(-1, 1, v, M); } void SubRow(const Tensor &v, Tensor *M) { AddRow(-1, 1, v, M); } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/concat.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/concat.cc b/src/model/layer/concat.cc new file mode 100644 index 0000000..b1c0b11 --- /dev/null +++ b/src/model/layer/concat.cc @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/model/layer.h" +#include "./concat.h" +namespace singa { + +RegisterLayerClass(singa_concat, Concat); +RegisterLayerClass(singacpp_concat, Concat); +RegisterLayerClass(singacuda_concat, Concat); +RegisterLayerClass(singacl_concat, Concat); + +void Concat::Setup(const vector<Shape>& in_shapes, const LayerConf& conf) { + Layer::Setup(in_shapes, conf); + dim_size_.clear(); + axis_ = conf.concat_conf().axis(); + out_sample_shape_ = {0, 0}; + out_sample_shape_[1 - axis_] = in_shapes[0][1 - axis_]; + for (auto& s: in_shapes) { + out_sample_shape_[axis_] += s[axis_]; + dim_size_.push_back(s[axis_]); + // LOG(ERROR) << s[axis_]; + } +} + +const vector<Tensor> Concat::Forward(int flag, const vector<Tensor>& inputs) { + vector<Tensor> outputs; + if (inputs.size() == 1u) { + outputs = inputs; + } else { + if(axis_ == 0) + outputs.push_back(ConcatRows(inputs)); + else + outputs.push_back(ConcatColumns(inputs)); + } + return outputs; +} + +const std::pair<vector<Tensor>, vector<Tensor>> Concat::Backward( + int flag, const vector<Tensor>& grads) { + vector<Tensor> input_grad, param_grad; + CHECK_EQ(grads.size(), 1u) << "Concat layer only have one output tensor."; + for (size_t i = 0, offset = 0; i < dim_size_.size(); i++) { + if (axis_ == 0) + input_grad.push_back(SliceRows(grads.at(0), offset, + offset + dim_size_[i])); + else + input_grad.push_back(SliceColumns(grads.at(0), offset, + offset + dim_size_[i])); + offset += dim_size_[i]; + } + return std::make_pair(input_grad, param_grad); +} + +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/concat.h ---------------------------------------------------------------------- diff --git a/src/model/layer/concat.h b/src/model/layer/concat.h new file mode 100644 index 0000000..59293d7 --- /dev/null +++ b/src/model/layer/concat.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SINGA_MODEL_LAYER_CONCAT_H_ +#define SINGA_MODEL_LAYER_CONCAT_H_ +#include <utility> +#include <string> +#include <vector> +#include "singa/model/layer.h" + +namespace singa { +class Concat : public Layer { + public: + /// \copydoc Layer::layer_type() + // const std::string layer_type() const override { return "Concat"; } + + /// \copydoc Layer::Setup(const LayerConf&); + void Setup(const vector<Shape>& in_shapes, const LayerConf& conf); + const Shape GetOutputSampleShape() const override { + CHECK(out_sample_shape_.size()) << "You may haven't call Setup()"; + return out_sample_shape_; + } + + /// \copydoc Layer::Forward(int flag, const Tensor&) + const vector<Tensor> Forward(int flag, const vector<Tensor>& input) override; + + /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&); + const std::pair<vector<Tensor>, vector<Tensor>> Backward(int flag, + const vector<Tensor>& grad) override; + + protected: + /// 0 for concat rows; 1 for concat cols + int axis_ = 0; + /// dim_size_[i] the size of the i-th source tensor on the concat dim + vector<int> dim_size_; + Shape out_sample_shape_; +}; + +} // namespace singa +#endif // SINGA_MODEL_LAYER_CONCAT_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/cudnn_rnn.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc index 0788801..583dcda 100644 --- a/src/model/layer/cudnn_rnn.cc +++ b/src/model/layer/cudnn_rnn.cc @@ -55,6 +55,7 @@ void CudnnRNN::ToDevice(std::shared_ptr<Device> device) { RNN::ToDevice(device); workspace_.ToDevice(device); reserve_space_.ToDevice(device); + dropout_state_.ToDevice(device); } void CudnnRNN::DestroyIODescriptors() { @@ -281,6 +282,23 @@ const vector<Tensor> CudnnRNN::Forward(int flag, const vector<Tensor> &inputs) { cy.ResetLike(hy); } + int did = input.device()->id(); + CHECK_EQ(did, output.device()->id()); + if (hx.Size()) { + CHECK_EQ(did, hx.device()->id()); + CHECK_EQ(hx.device()->lang(), kCuda); + } + if (cx.Size()) { + CHECK_EQ(did, cx.device()->id()); + CHECK_EQ(cx.device()->lang(), kCuda); + } + CHECK_EQ(did, weight_.device()->id()); + CHECK_EQ(did, workspace_.device()->id()); + CHECK_EQ(input.device()->lang(), kCuda); + CHECK_EQ(output.device()->lang(), kCuda); + CHECK_EQ(weight_.device()->lang(), kCuda); + CHECK_EQ(workspace_.device()->lang(), kCuda); + // LOG(INFO) << "hidden size " << hy.Size(); // LOG(INFO) << "weight size " << weight_.Size() << " value " << weight_.L1(); Block *inb = input.block(), *outb = output.block(), @@ -289,6 +307,8 @@ const vector<Tensor> CudnnRNN::Forward(int flag, const vector<Tensor> &inputs) { *wspace = this->workspace_.block(), *rspace = this->reserve_space_.block(); if (flag & kTrain) { + CHECK_EQ(reserve_space_.device()->lang(), kCuda); + CHECK_EQ(did, reserve_space_.device()->id()); dev->Exec( [inb, outb, wb, hxb, cxb, hyb, cyb, wspace, rspace, this](Context *ctx) { // clang-format off http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/slice.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/slice.cc b/src/model/layer/slice.cc new file mode 100644 index 0000000..690a03e --- /dev/null +++ b/src/model/layer/slice.cc @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/model/layer.h" +#include "./slice.h" +namespace singa { + +RegisterLayerClass(singa_slice, Slice); +RegisterLayerClass(singacpp_slice, Slice); +RegisterLayerClass(singacuda_slice, Slice); +RegisterLayerClass(singacl_slice, Slice); + +void Slice::Setup(const Shape& in_sample, const LayerConf& conf) { + Layer::Setup(in_sample, conf); + out_sample_shapes_.clear(); + axis_ = conf.slice_conf().axis(); + int offset = 0; + // #slice point = # out tensors - 1 + for (size_t p : conf.slice_conf().slice_point()) { + Shape s{0, 0}; + s[1 - axis_] = in_sample[1 - axis_]; + s[axis_] = p - offset; + offset = p; + out_sample_shapes_.push_back(s); + } + Shape s{0, 0}; + s[1 - axis_] = in_sample[1 - axis_]; + s[axis_] = in_sample[axis_] - offset; + out_sample_shapes_.push_back(s); +} + +const vector<Tensor> Slice::Forward(int flag, const vector<Tensor>& inputs) { + vector<Tensor> outputs; + CHECK_EQ(inputs.size(), 1u) << "Split layer only have one input tensor."; + size_t offset = 0; + for (auto& s : out_sample_shapes_) { + if (axis_ == 0) + outputs.push_back(SliceRows(inputs.at(0), offset, offset + s[axis_])); + else + outputs.push_back(SliceColumns(inputs.at(0), offset, offset + s[axis_])); + offset += s[axis_]; + } + return outputs; +} + +const std::pair<vector<Tensor>, vector<Tensor>> Slice::Backward( + int flag, const vector<Tensor>& grads) { + vector<Tensor> input_grad, param_grad; + CHECK_EQ(grads.size(), out_sample_shapes_.size()); + if (axis_ == 0) + input_grad.push_back(ConcatRows(grads)); + else + input_grad.push_back(ConcatColumns(grads)); + return std::make_pair(input_grad, param_grad); +} + +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/slice.h ---------------------------------------------------------------------- diff --git a/src/model/layer/slice.h b/src/model/layer/slice.h new file mode 100644 index 0000000..99ce468 --- /dev/null +++ b/src/model/layer/slice.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SINGA_MODEL_LAYER_SLICE_H_ +#define SINGA_MODEL_LAYER_SLICE_H_ +#include <utility> +#include <string> +#include <vector> +#include "singa/model/layer.h" + +namespace singa { +class Slice : public Layer { + public: + /// \copydoc Layer::layer_type() + // const std::string layer_type() const override { return "Slice"; } + + /// \copydoc Layer::Setup(const LayerConf&); + void Setup(const Shape& in_sample, const LayerConf& conf) override; + /// the i-th subshape is the shape of the i-th output tensor + const Shape GetOutputSampleShape(int k) override { + CHECK(out_sample_shapes_.size()) << "You may haven't call Setup()"; + return out_sample_shapes_[k]; + } + + /// \copydoc Layer::Forward(int flag, const Tensor&) + const vector<Tensor> Forward(int flag, const vector<Tensor>& input) override; + + /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&); + const std::pair<vector<Tensor>, vector<Tensor>> Backward(int flag, + const vector<Tensor>& grad) override; + + protected: + /// 0 for slice rows; 1 for slice cols + int axis_ = 0; + /// out_sample_shapes_[i] is the shape of the i-th output tensor + vector<Shape> out_sample_shapes_; +}; + +} // namespace singa +#endif // SINGA_MODEL_LAYER_CONCAT_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/split.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/split.cc b/src/model/layer/split.cc index 6b38a2b..b0e35e6 100644 --- a/src/model/layer/split.cc +++ b/src/model/layer/split.cc @@ -21,6 +21,9 @@ namespace singa { RegisterLayerClass(singa_split, Split); +RegisterLayerClass(singacpp_split, Split); +RegisterLayerClass(singacuda_split, Split); +RegisterLayerClass(singacl_split, Split); void Split::Setup(const Shape& in_sample, const LayerConf& conf) { Layer::Setup(in_sample, conf); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/test/singa/test_concat.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_concat.cc b/test/singa/test_concat.cc new file mode 100644 index 0000000..80183a7 --- /dev/null +++ b/test/singa/test_concat.cc @@ -0,0 +1,193 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "../src/model/layer/concat.h" +#include "gtest/gtest.h" + +using singa::Shape; + +TEST(Concat, Setup) { + Shape s1 {2u, 3u}; + Shape s2 {1u, 3u}; + singa::LayerConf conf; + conf.set_type("singa_concat"); + conf.mutable_concat_conf()->set_axis(0); + singa::Concat layer; + layer.Setup({s1, s2}, conf); + auto s = layer.GetOutputSampleShape(); + EXPECT_EQ(s[0], 3u); + EXPECT_EQ(s[1], 3u); +} + +void ForwardConcatRowTest(std::shared_ptr<singa::Device> dev) { + size_t a = 2u, b = 1u, c = 3u; + singa::Tensor t1({a, c}, dev); + singa::Tensor t2({b, c}, dev); + singa::LayerConf conf; + conf.set_type("singa_concat"); + conf.mutable_concat_conf()->set_axis(0); + singa::Concat layer; + layer.Setup({t1.shape(), t2.shape()}, conf); + layer.ToDevice(dev); + + t1.SetValue(1.0f); + t2.SetValue(2.0f); + auto out = layer.Forward(singa::kTrain, {t1, t2}); + EXPECT_EQ(out.size(), 1u); + + out[0].ToHost(); + const float * outptr = out[0].data<float>(); + for (size_t i = 0; i < a; i++) { + for (size_t j = 0; j < c; j++) + EXPECT_FLOAT_EQ(outptr[i * c + j], 1.0f); + } + for (size_t i = a; i < a + b; i++) { + for (size_t j = 0; j < c; j++) + EXPECT_FLOAT_EQ(outptr[i * c + j], 2.0f); + } + +} + +void ForwardConcatColumnTest(std::shared_ptr<singa::Device> dev) { + size_t a = 2u, b = 1u, c = 3u; + singa::Tensor t1({c, a}, dev); + singa::Tensor t2({c, b}, dev); + singa::LayerConf conf; + conf.set_type("singa_concat"); + conf.mutable_concat_conf()->set_axis(1); + singa::Concat layer; + layer.Setup({t1.shape(), t2.shape()}, conf); + layer.ToDevice(dev); + + t1.SetValue(1.0f); + t2.SetValue(2.0f); + auto out = layer.Forward(singa::kTrain, {t1, t2}); + EXPECT_EQ(out.size(), 1u); + out[0].ToHost(); + const float * outptr = out[0].data<float>(); + for (size_t i = 0; i < c; i++) { + for (size_t j = 0; j < a; j++) + EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 1.0f); + } + for (size_t i = 0; i < c; i++) { + for (size_t j = a; j < a + b; j++) + EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 2.0f); + } + +} +TEST(Concat, ForwardConcatRowCpp) { + ForwardConcatRowTest(singa::defaultDevice); +} + +TEST(Concat, ForwardConcatColumnCpp) { + ForwardConcatColumnTest(singa::defaultDevice); +} + + +#ifdef USE_CUDA +TEST(Concat, ForwardConcatRowCuda) { + ForwardConcatRowTest(std::make_shared<singa::CudaGPU>()); +} + +TEST(Concat, ForwardConcatColumnCuda) { + ForwardConcatColumnTest(std::make_shared<singa::CudaGPU>()); +} +#endif // USE_CUDA + + +void BackwardConcatRowTest(std::shared_ptr<singa::Device> dev) { + size_t a = 2u, b = 1u, c = 3u; + singa::LayerConf conf; + conf.set_type("singa_concat"); + conf.mutable_concat_conf()->set_axis(0); + singa::Concat layer; + layer.Setup({{a, c}, {b, c}}, conf); + layer.ToDevice(dev); + + singa::Tensor t({a + b, c}, dev); + singa::Uniform(-1.f, 1.f, &t); + auto out = layer.Backward(singa::kTrain, {t}); + auto grads = out.first; + EXPECT_EQ(grads.size(), 2u); + + t.ToHost(); + const float* tptr = t.data<float>(); + + grads[0].ToHost(); + const float * outa = grads[0].data<float>(); + for (size_t i = 0; i < a; i++) + for (size_t j = 0; j < c; j++) + EXPECT_FLOAT_EQ(outa[i * c + j], tptr[i * c + j]); + grads[1].ToHost(); + const float * outb = grads[1].data<float>(); + for (size_t i = 0; i < b; i++) + for (size_t j = 0; j < c; j++) + EXPECT_FLOAT_EQ(outb[i * c + j], tptr[(i + a) * c + j]); +} + +void BackwardConcatColumnTest(std::shared_ptr<singa::Device> dev) { + size_t a = 2u, b = 1u, c = 3u; + singa::LayerConf conf; + conf.set_type("singa_concat"); + conf.mutable_concat_conf()->set_axis(1); + singa::Concat layer; + layer.Setup({{c, a}, {c, b}}, conf); + layer.ToDevice(dev); + + singa::Tensor t({c, a + b}, dev); + singa::Uniform(-1.f, 1.f, &t); + auto out = layer.Backward(singa::kTrain, {t}); + auto grads = out.first; + EXPECT_EQ(grads.size(), 2u); + + t.ToHost(); + const float* tptr = t.data<float>(); + + grads[0].ToHost(); + const float * outa = grads[0].data<float>(); + for (size_t i = 0; i < c; i++) + for (size_t j = 0; j < a; j++) + EXPECT_FLOAT_EQ(outa[i * a + j], tptr[i * (a + b) + j]); + grads[1].ToHost(); + const float * outb = grads[1].data<float>(); + for (size_t i = 0; i < c; i++) + for (size_t j = 0; j < b; j++) + EXPECT_FLOAT_EQ(outb[i * b + j], tptr[i * (a + b) + a + j]); +} + +TEST(Concat, BackwardConcatRowCpp) { + BackwardConcatRowTest(singa::defaultDevice); +} + +TEST(Concat, BackwardConcatColumn) { + BackwardConcatColumnTest(singa::defaultDevice); +} + + +#ifdef USE_CUDA +TEST(Concat, BackwardConcatRowCuda) { + BackwardConcatRowTest(std::make_shared<singa::CudaGPU>()); +} + +TEST(Concat, BackwardConcatColumnCuda) { + BackwardConcatColumnTest(std::make_shared<singa::CudaGPU>()); +} +#endif // USE_CUDA
