SINGA-190 - Add prelu layer and flatten layer Implement prelu layer and flatten layer for cpu version.
Write gtest for prelu and flatten layer. Pass all tests. Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5afd81b7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5afd81b7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5afd81b7 Branch: refs/heads/master Commit: 5afd81b7f4841b15ce292b5b2e3e26c25a79b912 Parents: 04e23d1 Author: jixin <[email protected]> Authored: Wed Jun 8 15:58:09 2016 +0800 Committer: jixin <[email protected]> Committed: Sat Jun 11 16:46:59 2016 +0800 ---------------------------------------------------------------------- src/model/layer/flatten.cc | 62 ++++++++ src/model/layer/flatten.h | 54 +++++++ src/model/layer/prelu.cc | 169 +++++++++++++++++++++ src/model/layer/prelu.h | 60 ++++++++ src/proto/model.proto | 321 ++++++++++++++++++---------------------- test/singa/test_flatten.cc | 156 +++++++++++++++++++ test/singa/test_prelu.cc | 149 +++++++++++++++++++ 7 files changed, 793 insertions(+), 178 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/flatten.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/flatten.cc b/src/model/layer/flatten.cc new file mode 100644 index 0000000..3ed37fe --- /dev/null +++ b/src/model/layer/flatten.cc @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/model/layer.h" +#include "./flatten.h" +namespace singa { + +void Flatten::Setup(const LayerConf &conf) { + Layer::Setup(conf); + axis_ = conf.flatten_conf().axis(); +} + +const Tensor Flatten::Forward(int flag, const Tensor &input) { + Tensor output = input; + input_shape_ = input.shape(); + if (!Axis()) { + // reshape to 1D + size_t dim = output.Size(); + output.Reshape(Shape { + dim + }); + output_shape_ = Shape { dim } + ; + } else { + // reshape to 2D + size_t dim1 = 1, dim2; + for (int i = 0; i < Axis(); i++) + dim1 *= output.shape(i); + dim2 = output.Size() / dim1; + output.Reshape(Shape { + dim1, dim2 + }); + output_shape_ = Shape { dim1, dim2 } + ; + } + return output; +} + +const std::pair<Tensor, vector<Tensor> > Flatten::Backward(int flag, + const Tensor &grad) { + vector<Tensor> param_grad; + Tensor input_grad = grad; + input_grad.Reshape(Input_shape()); + return std::make_pair(input_grad, param_grad); +} + +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/flatten.h ---------------------------------------------------------------------- diff --git a/src/model/layer/flatten.h b/src/model/layer/flatten.h new file mode 100644 index 0000000..cb36542 --- /dev/null +++ b/src/model/layer/flatten.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SRC_MODEL_LAYER_FLATTEN_H_ +#define SRC_MODEL_LAYER_FLATTEN_H_ +#include <utility> +#include <string> +#include <vector> +#include "singa/model/layer.h" + +namespace singa { +class Flatten : public Layer { +public: + /// \copydoc Layer::layer_type(); + const std::string layer_type() const override { return "Flatten"; } + + /// \copydoc Layer::Setup(const LayerConf&); + void Setup(const LayerConf &conf) override; + + /// \copydoc Layer::Forward(int flag, const Tensor&); + const Tensor Forward(int flag, const Tensor &input) override; + + /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&); + const std::pair<Tensor, vector<Tensor> > Backward(int flag, + const Tensor &grad) + override; + + const int Axis() const { return axis_; } + const Shape Input_shape() const { return input_shape_; } + const Shape Output_shape() const { return output_shape_; } + +protected: + /// flatten layer reshape the input to 2D, one from 0 to axis_-1, one from + /// axis_ to end. + /// if axis_ is 0, reshape the input to 1D. + int axis_; + Shape input_shape_, output_shape_; +}; +} // namespace singa +#endif // SRC_MODEL_LAYER_FLATTEN_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/prelu.cc ---------------------------------------------------------------------- diff --git a/src/model/layer/prelu.cc b/src/model/layer/prelu.cc new file mode 100644 index 0000000..1d6a2e7 --- /dev/null +++ b/src/model/layer/prelu.cc @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/model/layer.h" +#include "./prelu.h" +namespace singa { + +void PReLU::Setup(const LayerConf &conf) { + Layer::Setup(conf); + channel_shared_ = conf.prelu_conf().channel_shared(); + format_ = conf.prelu_conf().format(); + // Push back params into param_values_ + for (const auto &spec : conf.param()) + param_specs_.push_back(spec); + param_values_.push_back(&a_); +} + +const Tensor PReLU::Forward(int flag, const Tensor &input) { + Tensor output; + if (!channel_shared_) { + size_t n, c, h, w; + Tensor temp = (input <= 0.f); + if (temp.nDim() == 4) { + if (format_ == "NCHW") { + n = temp.shape(0); + c = temp.shape(1); + h = temp.shape(2); + w = temp.shape(3); + temp.Reshape(Shape { + n *c, h *w + }); + Tensor temp_a(Shape { + n, c + }); + Uniform(1.f, 1.f, &temp_a); + MultRow(a_, &temp_a); + temp_a.Reshape(Shape { + n *c + }); + MultColumn(temp_a, &temp); + } else if (format_ == "NHWC") { + n = temp.shape(0); + h = temp.shape(1); + w = temp.shape(2); + c = temp.shape(3); + temp.Reshape(Shape { + n *h *w, c + }); + MultRow(a_, &temp); + } else { + LOG(FATAL) << "Incorrect input format for prelu layer."; + } + } else { + LOG(FATAL) << "Incorrect input format for prelu layer."; + } + output = input * ((input > 0.f) + temp); + } else { + // share the first param of Tensor A along all channels + const float a = a_.data<const float *>()[0]; + output = input * ((input > 0.f) + (input <= 0.f) * a); + } + if (flag & kTrain) + buf_.push(input); + return output; +} + +const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag, + const Tensor &grad) { + vector<Tensor> param_grad; + CHECK(!buf_.empty()); + Tensor input_grad, input = buf_.top(); + buf_.pop(); + Tensor da; + da.ResetLike(a_); + if (!channel_shared_) { + size_t n, c, h, w; + Tensor temp1 = (input <= 0.f); + if (temp1.nDim() == 4) { + if (format_ == "NCHW") { + n = temp1.shape(0); + c = temp1.shape(1); + h = temp1.shape(2); + w = temp1.shape(3); + temp1.Reshape(Shape { + n *c, h *w + }); + Tensor temp_a(Shape { + n, c + }); + Uniform(1.f, 1.f, &temp_a); + MultRow(a_, &temp_a); + temp_a.Reshape(Shape { + n *c + }); + MultColumn(temp_a, &temp1); + temp1.Reshape(Shape { + n, c, h, w + }); + } else if (format_ == "NHWC") { + n = temp1.shape(0); + h = temp1.shape(1); + w = temp1.shape(2); + c = temp1.shape(3); + temp1.Reshape(Shape { + n *h *w, c + }); + MultRow(a_, &temp1); + temp1.Reshape(Shape { + n, h, w, c + }); + } else { + LOG(FATAL) << "Incorrect input format for prelu layer."; + } + } else { + LOG(FATAL) << "Incorrect input format for prelu layer."; + } + input_grad = grad * input * ((input > 0.f) + temp1); + Tensor temp2 = grad * input * (input <= 0.f), temp3(Shape { + n *c + }); + if (format_ == "NCHW") { + temp2.Reshape(Shape { + n *c, h *w + }); + SumColumns(temp2, &temp3); + temp3.Reshape(Shape { + n, c + }); + SumRows(temp3, &da); + } else if (format_ == "NHWC") { + temp2.Reshape(Shape { + n *h *w, c + }); + SumRows(temp2, &da); + } + } else { + // share the first param of Tensor A along all channels + const float a = a_.data<const float *>()[0]; + input_grad = grad * input * ((input > 0.f) + (input <= 0.f) * a); + Tensor temp = grad * input * (input <= 0.f); + float sum = Sum<float>(temp); + Uniform(1.f, 1.f, &da); + da *= sum; + } + param_grad.push_back(da); + return std::make_pair(input_grad, param_grad); +} + +void PReLU::ToDevice(Device *device) { + Layer::ToDevice(device); + a_.ToDevice(device); +} + +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/prelu.h ---------------------------------------------------------------------- diff --git a/src/model/layer/prelu.h b/src/model/layer/prelu.h new file mode 100644 index 0000000..1a01d98 --- /dev/null +++ b/src/model/layer/prelu.h @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SINGA_MODEL_LAYER_PRELU_H_ +#define SINGA_MODEL_LAYER_PRELU_H_ +#include <utility> +#include <string> +#include <vector> +#include "singa/model/layer.h" + +namespace singa { +class PReLU : public Layer { + public: + /// \copydoc Layer::layer_type() + const std::string layer_type() const override { return "PReLU"; } + + /// \copydoc Layer::Setup(const LayerConf&); + void Setup(const LayerConf &conf) override; + + /// \copydoc Layer::Forward(int flag, const Tensor&) + const Tensor Forward(int flag, const Tensor &input) override; + + /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&); + const std::pair<Tensor, vector<Tensor> > Backward(int flag, + const Tensor &grad) + override; + + void ToDevice(Device *device); + + const bool Channel_shared() const { return channel_shared_; } + const Tensor A() const { return a_; } + const std::string Format() const { return format_; } + + void Set_a(Tensor a) { + a_.ResetLike(a); + a_.CopyData(a); + } + + protected: + bool channel_shared_; + std::string format_; // format_ has two valid value, i.e. NCHW, NHWC + Tensor a_; // shape of a_ is 2D, i.e. (channels, 1) + std::stack<Tensor> buf_; +}; +} // namespace singa +#endif // SINGA_MODEL_LAYER_PRELU_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/proto/model.proto ---------------------------------------------------------------------- diff --git a/src/proto/model.proto b/src/proto/model.proto index d368296..1d1f3cf 100644 --- a/src/proto/model.proto +++ b/src/proto/model.proto @@ -33,64 +33,59 @@ package singa; /// using Python (or C++/Java). // Specifies the shape (dimensions) of a Blob. -message BlobShape { - repeated int64 dim = 1 [packed = true]; -} +message BlobShape { repeated int64 dim = 1[packed = true]; } message BlobProto { optional BlobShape shape = 7; - repeated float data = 5 [packed = true]; - repeated float diff = 6 [packed = true]; - repeated double double_data = 8 [packed = true]; - repeated double double_diff = 9 [packed = true]; + repeated float data = 5[packed = true]; + repeated float diff = 6[packed = true]; + repeated double double_data = 8[packed = true]; + repeated double double_diff = 9[packed = true]; // 4D dimensions -- deprecated. Use "shape" instead. - optional int32 num = 1 [default = 0]; - optional int32 channels = 2 [default = 0]; - optional int32 height = 3 [default = 0]; - optional int32 width = 4 [default = 0]; + optional int32 num = 1[default = 0]; + optional int32 channels = 2[default = 0]; + optional int32 height = 3[default = 0]; + optional int32 width = 4[default = 0]; } message FillerConf { // The filler type, case insensitive - optional string type = 1 [default = 'constant']; - optional float value = 2 [default = 0]; // the value in constant filler - optional float min = 3 [default = 0]; // the min value in uniform filler - optional float max = 4 [default = 1]; // the max value in uniform filler - optional float mean = 5 [default = 0]; // the mean value in Gaussian filler - optional float std = 6 [default = 1]; // the std value in Gaussian filler + optional string type = 1[default = 'constant']; + optional float value = 2[default = 0]; // the value in constant filler + optional float min = 3[default = 0]; // the min value in uniform filler + optional float max = 4[default = 1]; // the max value in uniform filler + optional float mean = 5[default = 0]; // the mean value in Gaussian filler + optional float std = 6[default = 1]; // the std value in Gaussian filler // The expected number of non-zero output weights for a given input in // Gaussian filler -- the default -1 means don't perform sparsification. /* optional int32 sparse = 7 [default = -1]; */ // Normalize the filler variance by fan_in, fan_out, or their average. // Applies to 'xavier' and 'msra' fillers. enum VarianceNorm { - FAN_IN = 0; - FAN_OUT = 1; - AVERAGE = 2; - } - optional VarianceNorm variance_norm = 8 [default = FAN_IN]; + FAN_IN = 0; FAN_OUT = 1; AVERAGE = 2; + } optional VarianceNorm variance_norm = 8[default = FAN_IN]; } /// SINGA message message OptimizerConf { // case insensitive - optional string type = 1 [default = "sgd"]; + optional string type = 1[default = "sgd"]; // used by RMSprop and Adadelta - optional float rho = 2 [default = 0.001]; + optional float rho = 2[default = 0.001]; // used by Adam and AdamMax - optional float beta_1 = 3 [default = 0.9]; - optional float beta_2 = 4 [default = 0.999]; + optional float beta_1 = 3[default = 0.9]; + optional float beta_2 = 4[default = 0.999]; // used by vanilla sgd and nesterov - optional float momentum = 5 [default = 0.9]; + optional float momentum = 5[default = 0.9]; } message ConstraintConf { // case insensitive to limit the parameter value/gradient scale - optional string type = 1 [default = "l2"]; + optional string type = 1[default = "l2"]; // e.g., the threshold for limiting the parameter scale. optional float threshold = 2; } @@ -98,7 +93,7 @@ message ConstraintConf { /// SINGA message message RegularizerConf { // case insensitive to regularize the parameters, e.g., L2. - optional string type = 1 [default = "l2"]; + optional string type = 1[default = "l2"]; // e.g., the weight decay for L2 regularizer optional float coefficient = 2; } @@ -124,10 +119,10 @@ message ParamSpec { */ // The multiplier on the global learning rate for this parameter. - optional float lr_mult = 3 [default = 1.0]; + optional float lr_mult = 3[default = 1.0]; // The multiplier on the global weight decay for this parameter. - optional float decay_mult = 4 [default = 1.0]; + optional float decay_mult = 4[default = 1.0]; // SINGA uses this filed internally. Users just configure the fillers in // Layer specific conf message as caffe (style). @@ -137,14 +132,13 @@ message ParamSpec { } enum Phase { - kTrain = 4; - kEval = 8; -} -// NOTE -// Update the next available ID when you add a new LayerConf field. -// -// LayerConf next available layer-specific ID: 139 (last added: tile_param) -message LayerConf { + kTrain = 4; kEval = 8; +} + // NOTE + // Update the next available ID when you add a new LayerConf field. + // + // LayerConf next available layer-specific ID: 139 (last added: tile_param) + message LayerConf { optional string name = 1; // the layer name optional string type = 2; // the layer type /* repeated string bottom = 3; // the name of each bottom blob */ @@ -248,7 +242,8 @@ message TransformationConf { optional uint32 crop_size = 3 [default = 0]; // mean_file and mean_value cannot be specified at the same time optional string mean_file = 4; - // if specified can be repeated once (would substract it from all the channels) + // if specified can be repeated once (would substract it from all the +channels) // or can be repeated the same number of times as channels // (would subtract them from the corresponding channel) repeated float mean_value = 5; @@ -265,34 +260,33 @@ message LossConf { optional int32 ignore_label = 1; // If true, normalize each batch across all instances (including spatial // dimesions, but not ignored instances); else, divide by batch size only. - optional bool normalize = 2 [default = true]; + optional bool normalize = 2[default = true]; } message MetricConf { // When computing accuracy, count as correct by comparing the true label to // the top k scoring classes. By default, only compare to the top scoring // class (i.e. argmax). - optional uint32 top_k = 1 [default = 1]; + optional uint32 top_k = 1[default = 1]; // The "label" axis of the prediction blob, whose argmax corresponds to the // predicted label -- may be negative to index from the end (e.g., -1 for the // last axis). For example, if axis == 1 and the predictions are // (N x C x H x W), the label blob is expected to contain N*H*W ground truth // labels with integer values in {0, 1, ..., C-1}. - optional int32 axis = 2 [default = 1]; + optional int32 axis = 2[default = 1]; // If specified, ignore instances with the given label. optional int32 ignore_label = 3; } -// Messages that store hyper-parameters used by individual layer types follow, in +// Messages that store hyper-parameters used by individual layer types follow, +// in // alphabetical order. - - message ArgMaxConf { // If true produce pairs (argmax, maxval) - optional bool out_max_val = 1 [default = false]; - optional uint32 top_k = 2 [default = 1]; + optional bool out_max_val = 1[default = false]; + optional uint32 top_k = 2[default = 1]; // The axis along which to maximise -- may be negative to index from the // end (e.g., -1 for the last axis). // By default ArgMaxLayer maximizes over the flattened trailing dimensions @@ -305,54 +299,51 @@ message ConcatConf { // end (e.g., -1 for the last axis). Other axes must have the // same dimension for all the bottom blobs. // By default, ConcatLayer concatenates blobs along the "channels" axis (1). - optional int32 axis = 2 [default = 1]; + optional int32 axis = 2[default = 1]; // DEPRECATED: alias for "axis" -- does not support negative indexing. - optional uint32 concat_dim = 1 [default = 1]; + optional uint32 concat_dim = 1[default = 1]; } message ContrastiveLossConf { // margin for dissimilar pair - optional float margin = 1 [default = 1.0]; + optional float margin = 1[default = 1.0]; // The first implementation of this cost did not exactly match the cost of // Hadsell et al 2006 -- using (margin - d^2) instead of (margin - d)^2. // legacy_version = false (the default) uses (margin - d)^2 as proposed in the // Hadsell paper. New models should probably use this version. // legacy_version = true uses (margin - d^2). This is kept to support / // reproduce existing models and results - optional bool legacy_version = 2 [default = false]; + optional bool legacy_version = 2[default = false]; } message ConvolutionConf { optional uint32 num_output = 1; // The number of outputs for the layer - optional bool bias_term = 2 [default = true]; // whether to have bias terms + optional bool bias_term = 2[default = true]; // whether to have bias terms // Pad, kernel size, and stride are all given as a single value for equal // dimensions in all spatial dimensions, or once per spatial dimension. - repeated uint32 pad = 3; // The padding size; defaults to 0 + repeated uint32 pad = 3; // The padding size; defaults to 0 repeated uint32 kernel_size = 4; // The kernel size - repeated uint32 stride = 6; // The stride; defaults to 1 + repeated uint32 stride = 6; // The stride; defaults to 1 // For 2D convolution only, the *_h and *_w versions may also be used to // specify both spatial dimensions. - optional uint32 pad_h = 9 [default = 0]; // The padding height (2D only) - optional uint32 pad_w = 10 [default = 0]; // The padding width (2D only) - optional uint32 kernel_h = 11; // The kernel height (2D only) - optional uint32 kernel_w = 12; // The kernel width (2D only) - optional uint32 stride_h = 13; // The stride height (2D only) - optional uint32 stride_w = 14; // The stride width (2D only) + optional uint32 pad_h = 9[default = 0]; // The padding height (2D only) + optional uint32 pad_w = 10[default = 0]; // The padding width (2D only) + optional uint32 kernel_h = 11; // The kernel height (2D only) + optional uint32 kernel_w = 12; // The kernel width (2D only) + optional uint32 stride_h = 13; // The stride height (2D only) + optional uint32 stride_w = 14; // The stride width (2D only) // SINGA: not supported. // optional uint32 group = 5 [default = 1]; // The group size for group conv optional FillerConf weight_filler = 7; // The filler for the weight - optional FillerConf bias_filler = 8; // The filler for the bias + optional FillerConf bias_filler = 8; // The filler for the bias enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 15 [default = DEFAULT]; + DEFAULT = 0; CAFFE = 1; CUDNN = 2; + } optional Engine engine = 15[default = DEFAULT]; // The axis to interpret as "channels" when performing convolution. // Preceding dimensions are treated as independent inputs; @@ -374,13 +365,12 @@ message ConvolutionConf { // SINGA: not supported; // optional bool force_nd_im2col = 17 [default = false]; - // SINGA: add by xiangrui // cudnn workspace size in MB - optional int32 workspace_byte_limit = 50 [default = 512]; + optional int32 workspace_byte_limit = 50[default = 512]; // cudnn algorithm preference // options: "fastest", "limited_workspace", "no_workspace" - optional string prefer = 51 [default = "fastest"]; + optional string prefer = 51[default = "fastest"]; // input shape optional int32 channels = 52; optional int32 height = 53; @@ -424,7 +414,7 @@ message DataConf { */ message DropoutConf { - optional float dropout_ratio = 1 [default = 0.5]; // dropout ratio + optional float dropout_ratio = 1[default = 0.5]; // dropout ratio } // DummyDataLayer fills any number of arbitrarily shaped blobs with random @@ -448,16 +438,13 @@ message DummyDataConf { message EltwiseConf { enum EltwiseOp { - PROD = 0; - SUM = 1; - MAX = 2; - } - optional EltwiseOp operation = 1 [default = SUM]; // element-wise operation + PROD = 0; SUM = 1; MAX = 2; + } optional EltwiseOp operation = 1[default = SUM]; // element-wise operation repeated float coeff = 2; // blob-wise coefficient for SUM operation // Whether to use an asymptotically slower (for >2 inputs) but stabler method // of computing the gradient for the PROD operation. (No effect for SUM op.) - optional bool stable_prod_grad = 3 [default = true]; + optional bool stable_prod_grad = 3[default = true]; } // Message that stores hyper-parameters used by EmbedLayer @@ -468,9 +455,9 @@ message EmbedConf { // 1 greater than the maximum possible input value. optional uint32 input_dim = 2; - optional bool bias_term = 3 [default = true]; // Whether to use a bias term - optional FillerConf weight_filler = 4; // The filler for the weight - optional FillerConf bias_filler = 5; // The filler for the bias + optional bool bias_term = 3[default = true]; // Whether to use a bias term + optional FillerConf weight_filler = 4; // The filler for the weight + optional FillerConf bias_filler = 5; // The filler for the bias } @@ -479,21 +466,21 @@ message ExpConf { // ExpLayer computes outputs y = base ^ (shift + scale * x), for base > 0. // Or if base is set to the default (-1), base is set to e, // so y = exp(shift + scale * x). - optional float base = 1 [default = -1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; + optional float base = 1[default = -1.0]; + optional float scale = 2[default = 1.0]; + optional float shift = 3[default = 0.0]; } /// Message that stores hyper-parameters used by FlattenLayer message FlattenConf { // The first axis to flatten: all preceding axes are retained in the output. // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 1 [default = 1]; + optional int32 axis = 1[default = 1]; // The last axis to flatten: all following axes are retained in the output. // May be negative to index from the end (e.g., the default -1 for the last // axis). - optional int32 end_axis = 2 [default = -1]; + optional int32 end_axis = 2[default = -1]; } /* @@ -519,11 +506,10 @@ message HDF5OutputConf { message HingeLossConf { enum Norm { - L1 = 1; - L2 = 2; + L1 = 1; L2 = 2; } - // Specify the Norm to use L1 or L2 - optional Norm norm = 1 [default = L1]; + // Specify the Norm to use L1 or L2 + optional Norm norm = 1[default = L1]; } /* @@ -566,29 +552,29 @@ message InfogainLossConf { message InnerProductConf { optional uint32 num_output = 1; // The number of outputs for the layer - optional bool bias_term = 2 [default = true]; // whether to have bias terms - optional FillerConf weight_filler = 3; // The filler for the weight - optional FillerConf bias_filler = 4; // The filler for the bias + optional bool bias_term = 2[default = true]; // whether to have bias terms + optional FillerConf weight_filler = 3; // The filler for the weight + optional FillerConf bias_filler = 4; // The filler for the bias // The first axis to be lumped into a single inner product computation; // all preceding axes are retained in the output. // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 5 [default = 1]; + optional int32 axis = 5[default = 1]; } message DenseConf { optional uint32 num_output = 1; // The number of outputs for the layer - optional bool bias_term = 2 [default = true]; // whether to have bias terms - optional FillerConf weight_filler = 3; // The filler for the weight - optional FillerConf bias_filler = 4; // The filler for the bias + optional bool bias_term = 2[default = true]; // whether to have bias terms + optional FillerConf weight_filler = 3; // The filler for the weight + optional FillerConf bias_filler = 4; // The filler for the bias // The first axis to be lumped into a single inner product computation; // all preceding axes are retained in the output. // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 5 [default = 1]; + optional int32 axis = 5[default = 1]; optional uint32 num_input = 20; // The number of inputs for the layer - optional bool transpose = 21 [default = false]; // whether transpose or not + optional bool transpose = 21[default = false]; // whether transpose or not } // Message that stores hyper-parameters used by LogLayer @@ -596,22 +582,20 @@ message LogConf { // LogLayer computes outputs y = log_base(shift + scale * x), for base > 0. // Or if base is set to the default (-1), base is set to e, // so y = ln(shift + scale * x) = log_e(shift + scale * x) - optional float base = 1 [default = -1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; + optional float base = 1[default = -1.0]; + optional float scale = 2[default = 1.0]; + optional float shift = 3[default = 0.0]; } // Message that stores hyper-parameters used by LRNLayer message LRNConf { - optional uint32 local_size = 1 [default = 5]; - optional float alpha = 2 [default = 1.]; - optional float beta = 3 [default = 0.75]; + optional uint32 local_size = 1[default = 5]; + optional float alpha = 2[default = 1.]; + optional float beta = 3[default = 0.75]; enum NormRegion { - ACROSS_CHANNELS = 0; - WITHIN_CHANNEL = 1; - } - optional NormRegion norm_region = 4 [default = ACROSS_CHANNELS]; - optional float k = 5 [default = 1.]; + ACROSS_CHANNELS = 0; WITHIN_CHANNEL = 1; + } optional NormRegion norm_region = 4[default = ACROSS_CHANNELS]; + optional float k = 5[default = 1.]; } message MemoryDataConf { @@ -623,33 +607,30 @@ message MemoryDataConf { message MVNConf { // This parameter can be set to false to normalize mean only - optional bool normalize_variance = 1 [default = true]; + optional bool normalize_variance = 1[default = true]; // This parameter can be set to true to perform DNN-like MVN - optional bool across_channels = 2 [default = false]; + optional bool across_channels = 2[default = false]; // Epsilon for not dividing by zero while normalizing variance - optional float eps = 3 [default = 1e-9]; + optional float eps = 3[default = 1e-9]; } message PoolingConf { enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional PoolMethod pool = 1 [default = MAX]; // The pooling method + MAX = 0; AVE = 1; STOCHASTIC = 2; + } optional PoolMethod pool = 1[default = MAX]; // The pooling method // Pad, kernel size, and stride are all given as a single value for equal // dimensions in height and width or as Y, X pairs. - optional uint32 pad = 4 [default = 0]; // The padding size (equal in Y, X) - optional uint32 pad_h = 9 [default = 0]; // The padding height - optional uint32 pad_w = 10 [default = 0]; // The padding width - optional uint32 kernel_size = 2; // The kernel size (square) - optional uint32 kernel_h = 5; // The kernel height - optional uint32 kernel_w = 6; // The kernel width - optional uint32 stride = 3 [default = 1]; // The stride (equal in Y, X) - optional uint32 stride_h = 7; // The stride height - optional uint32 stride_w = 8; // The stride width + optional uint32 pad = 4[default = 0]; // The padding size (equal in Y, X) + optional uint32 pad_h = 9[default = 0]; // The padding height + optional uint32 pad_w = 10[default = 0]; // The padding width + optional uint32 kernel_size = 2; // The kernel size (square) + optional uint32 kernel_h = 5; // The kernel height + optional uint32 kernel_w = 6; // The kernel width + optional uint32 stride = 3[default = 1]; // The stride (equal in Y, X) + optional uint32 stride_h = 7; // The stride height + optional uint32 stride_w = 8; // The stride width /* enum Engine { DEFAULT = 0; @@ -660,20 +641,20 @@ message PoolingConf { */ // If global_pooling then it will pool over the size of the bottom by doing // kernel_h = bottom->height and kernel_w = bottom->width - optional bool global_pooling = 12 [default = false]; + optional bool global_pooling = 12[default = false]; // Shape of source optional int32 channels = 50; optional int32 height = 51; optional int32 width = 52; // whether to propagate nan - optional bool nan_prop = 53 [default = false]; + optional bool nan_prop = 53[default = false]; } message PowerConf { // PowerLayer computes outputs y = (shift + scale * x) ^ power. - optional float power = 1 [default = 1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; + optional float power = 1[default = 1.0]; + optional float scale = 2[default = 1.0]; + optional float shift = 3[default = 0.0]; } /* message PythonConf { @@ -684,7 +665,8 @@ message PythonConf { // string, dictionary in Python dict format, JSON, etc. You may parse this // string in `setup` method and use it in `forward` and `backward`. optional string param_str = 3 [default = '']; - // Whether this PythonLayer is shared among worker solvers during data parallelism. + // Whether this PythonLayer is shared among worker solvers during data +parallelism. // If true, each worker solver sequentially run forward from this layer. // This value should be set true if you are using it as a data layer. optional bool share_in_parallel = 4 [default = false]; @@ -694,13 +676,8 @@ message PythonConf { // Message that stores hyper-parameters used by ReductionLayer message ReductionConf { enum ReductionOp { - SUM = 1; - ASUM = 2; - SUMSQ = 3; - MEAN = 4; - } - - optional ReductionOp operation = 1 [default = SUM]; // reduction operation + SUM = 1; ASUM = 2; SUMSQ = 3; MEAN = 4; + } optional ReductionOp operation = 1[default = SUM]; // reduction operation // The first axis to reduce to a scalar -- may be negative to index from the // end (e.g., -1 for the last axis). @@ -715,9 +692,9 @@ message ReductionConf { // If axis == 0 (the default), the output Blob always has the empty shape // (count 1), performing reduction across the entire input -- // often useful for creating new loss functions. - optional int32 axis = 2 [default = 0]; + optional int32 axis = 2[default = 0]; - optional float coeff = 3 [default = 1.0]; // coefficient for output + optional float coeff = 3[default = 1.0]; // coefficient for output } // Message that stores hyper-parameters used by ReLULayer @@ -727,7 +704,7 @@ message ReLUConf { // Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013). Rectifier nonlinearities // improve neural network acoustic models. In ICML Workshop on Deep Learning // for Audio, Speech, and Language Processing. - optional float negative_slope = 1 [default = 0]; + optional float negative_slope = 1[default = 0]; /* enum Engine { DEFAULT = 0; @@ -798,58 +775,50 @@ message ReshapeConf { // reshape_param { shape { dim: 2 dim: 1 dim: 8 } } // reshape_param { shape { dim: 1 } axis: 1 num_axes: 0 } // - optional int32 axis = 2 [default = 0]; - optional int32 num_axes = 3 [default = -1]; + optional int32 axis = 2[default = 0]; + optional int32 num_axes = 3[default = -1]; } message SigmoidConf { enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; + DEFAULT = 0; CAFFE = 1; CUDNN = 2; + } optional Engine engine = 1[default = DEFAULT]; } message SliceConf { // The axis along which to slice -- may be negative to index from the end // (e.g., -1 for the last axis). // By default, SliceLayer concatenates blobs along the "channels" axis (1). - optional int32 axis = 3 [default = 1]; + optional int32 axis = 3[default = 1]; repeated uint32 slice_point = 2; // DEPRECATED: alias for "axis" -- does not support negative indexing. - optional uint32 slice_dim = 1 [default = 1]; + optional uint32 slice_dim = 1[default = 1]; } -// Message that stores hyper-parameters used by SoftmaxLayer, SoftmaxWithLossLayer +// Message that stores hyper-parameters used by SoftmaxLayer, +// SoftmaxWithLossLayer message SoftmaxConf { enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; + DEFAULT = 0; CAFFE = 1; CUDNN = 2; + } optional Engine engine = 1[default = DEFAULT]; // The axis along which to perform the softmax -- may be negative to index // from the end (e.g., -1 for the last axis). // Any other axes will be evaluated as independent softmaxes. - optional int32 axis = 2 [default = 1]; + optional int32 axis = 2[default = 1]; } message TanHConf { enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 1 [default = DEFAULT]; + DEFAULT = 0; CAFFE = 1; CUDNN = 2; + } optional Engine engine = 1[default = DEFAULT]; } // Message that stores hyper-parameters used by TileLayer message TileConf { // The index of the axis to tile. - optional int32 axis = 1 [default = 1]; + optional int32 axis = 1[default = 1]; // The number of copies (tiles) of the blob to output. optional int32 tiles = 2; @@ -857,7 +826,7 @@ message TileConf { // Message that stores hyper-parameters used by ThresholdLayer message ThresholdConf { - optional float threshold = 1 [default = 0]; // Strictly positive values + optional float threshold = 1[default = 0]; // Strictly positive values } /* @@ -897,18 +866,12 @@ message WindowDataConf { message SPPConf { enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional uint32 pyramid_height = 1; - optional PoolMethod pool = 2 [default = MAX]; // The pooling method + MAX = 0; AVE = 1; STOCHASTIC = 2; + } optional uint32 pyramid_height = 1; + optional PoolMethod pool = 2[default = MAX]; // The pooling method enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 6 [default = DEFAULT]; + DEFAULT = 0; CAFFE = 1; CUDNN = 2; + } optional Engine engine = 6[default = DEFAULT]; } message PReLUConf { @@ -918,13 +881,15 @@ message PReLUConf { // Initial value of a_i. Default is a_i=0.25 for all i. optional FillerConf filler = 1; // Whether or not slope paramters are shared across channels. - optional bool channel_shared = 2 [default = false]; + optional bool channel_shared = 2[default = false]; + // format of the input. Default is NCHW. + optional string format = 50[default = "NCHW"]; } message BatchNormConf { // Used in the moving average computation runningMean = // newMean*factor + runningMean*(1-factor). - optional double factor = 1 [default = 0.9]; + optional double factor = 1[default = 0.9]; // input shape optional int32 channels = 2; optional int32 height = 3; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/test/singa/test_flatten.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_flatten.cc b/test/singa/test_flatten.cc new file mode 100644 index 0000000..906e4b8 --- /dev/null +++ b/test/singa/test_flatten.cc @@ -0,0 +1,156 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "../src/model/layer/flatten.h" +#include "gtest/gtest.h" + +using singa::Flatten; +TEST(Flatten, Setup) { + Flatten flt; + EXPECT_EQ("Flatten", flt.layer_type()); + + singa::LayerConf conf; + singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); + flattenconf->set_axis(1); + + flt.Setup(conf); + EXPECT_EQ(1, flt.Axis()); +} + +TEST(Flatten, ForwardCPU) { + const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f, + -2.f, -1.f }; + size_t n = sizeof(x) / sizeof(float); + singa::Shape s = { 2, 1, 3, 2 }; + singa::Tensor in(s); + in.CopyDataFromHostPtr<float>(x, n); + + int axis = 3; + Flatten flt; + singa::LayerConf conf; + singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); + flattenconf->set_axis(axis); + flt.Setup(conf); + + singa::Tensor out = flt.Forward(singa::kTrain, in); + EXPECT_EQ(n, out.Size()); + EXPECT_EQ(6, out.shape(0)); + EXPECT_EQ(2, out.shape(1)); + const float *yptr = out.data<const float *>(); + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(x[i], yptr[i]); +} + +TEST(Flatten, BackwardCPU) { + // directly use input as the output_grad for backward + // note that only the shape of input really matters + const float dy[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f, + -2.f, -1.f }; + size_t n = sizeof(dy) / sizeof(float); + singa::Tensor in(singa::Shape { + 2, 1, 3, 2 + }); + in.CopyDataFromHostPtr<float>(dy, n); + + int axis = 2; + Flatten flt; + singa::LayerConf conf; + singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); + flattenconf->set_axis(axis); + flt.Setup(conf); + + singa::Tensor temp = flt.Forward(singa::kTrain, in); + const auto out = flt.Backward(singa::kTrain, temp); + const float *xptr = out.first.data<const float *>(); + EXPECT_EQ(n, out.first.Size()); + EXPECT_EQ(2, out.first.shape(0)); + EXPECT_EQ(1, out.first.shape(1)); + EXPECT_EQ(3, out.first.shape(2)); + EXPECT_EQ(2, out.first.shape(3)); + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(dy[i], xptr[i]); +} + +#ifdef USE_CUDA +TEST(Flatten, ForwardGPU) { + const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f, + -2.f, -1.f }; + size_t n = sizeof(x) / sizeof(float); + singa::CudaGPU cuda(0, 1); + singa::Tensor in(singa::Shape { + 2, 1, 3, 2 + }, + &cuda); + in.CopyDataFromHostPtr<float>(x, n); + + int axis = 3; + Flatten flt; + singa::LayerConf conf; + singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); + flattenconf->set_axis(axis); + flt.Setup(conf); + + singa::Tensor out = flt.Forward(singa::kTrain, in); + singa::CppCPU host(0, 1); + out.ToDevice(&host); + EXPECT_EQ(n, out.Size()); + EXPECT_EQ(6, out.shape(0)); + EXPECT_EQ(2, out.shape(1)); + const float *yptr = out.data<const float *>(); + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(x[i], yptr[i]); +} + +TEST(Flatten, BackwardGPU) { + // directly use input as the output_grad for backward + // note that only the shape of input really matters + const float dy[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f, + -2.f, -1.f }; + size_t n = sizeof(dy) / sizeof(float); + singa::CudaGPU cuda(0, 1); + singa::Tensor in(singa::Shape { + 2, 1, 3, 2 + }, + &cuda); + in.CopyDataFromHostPtr<float>(dy, n); + + int axis = 2; + Flatten flt; + singa::LayerConf conf; + singa::FlattenConf *flattenconf = conf.mutable_flatten_conf(); + flattenconf->set_axis(axis); + flt.Setup(conf); + + singa::Tensor out = flt.Forward(singa::kTrain, in); + const auto ret = flt.Backward(singa::kTrain, out); + singa::CppCPU host(0, 1); + singa::Tensor in_diff = ret.first; + in_diff.ToDevice(&host); + const float *xptr = in_diff.data<const float *>(); + EXPECT_EQ(n, in_diff.Size()); + EXPECT_EQ(2, in_diff.shape(0)); + EXPECT_EQ(1, in_diff.shape(1)); + EXPECT_EQ(3, in_diff.shape(2)); + EXPECT_EQ(2, in_diff.shape(3)); + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(dy[i], xptr[i]); +} +#endif // USE_CUDA http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/test/singa/test_prelu.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_prelu.cc b/test/singa/test_prelu.cc new file mode 100644 index 0000000..2dde9e9 --- /dev/null +++ b/test/singa/test_prelu.cc @@ -0,0 +1,149 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "../src/model/layer/prelu.h" +#include "gtest/gtest.h" +#include "singa_config.h" + +using singa::PReLU; +TEST(PReLU, Setup) { + PReLU prelu; + EXPECT_EQ("PReLU", prelu.layer_type()); + + singa::LayerConf conf; + singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); + preluconf->set_channel_shared(true); + preluconf->set_format("NHWC"); + + prelu.Setup(conf); + EXPECT_EQ(true, prelu.Channel_shared()); + EXPECT_EQ("NHWC", prelu.Format()); +} + +TEST(PReLU, ForwardCPU) { + const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -1.f, -1.f, 2.f, -1.f, -2.f, + -2.f, -1.f }; + size_t n = sizeof(x) / sizeof(float); + size_t batchsize = 2, c = 3, h = 2, w = 1; + singa::Tensor in(singa::Shape { + batchsize, h, w, c + }); + in.CopyDataFromHostPtr<float>(x, n); + + PReLU prelu; + singa::LayerConf conf; + singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); + preluconf->set_channel_shared(false); + preluconf->set_format("NHWC"); + prelu.Setup(conf); + + const float neg_slope[] = { 0.25f, 0.5f, 0.75f }; + singa::Tensor a(singa::Shape { + c + }); + a.CopyDataFromHostPtr<float>(neg_slope, c); + prelu.Set_a(a); + + singa::Tensor out = prelu.Forward(singa::kTrain, in); + const float *yptr = out.data<const float *>(); + EXPECT_EQ(n, out.Size()); + + float *y = new float[n]; + size_t div_factor = prelu.Channel_shared() ? c : 1; + if (prelu.Format() == "NCHW") { + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f); + } + } else if (prelu.Format() == "NHWC") { + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f); + } + } + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(y[i], yptr[i]); +} + +TEST(PReLU, BackwardCPU) { + const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f, -1.f, 2.f, -1.f, -2.f, -2.f, -1.f}; + size_t n = sizeof(x) / sizeof(float); + size_t batchsize = 2, c = 3, h = 2, w = 1; + singa::Tensor in(singa::Shape { + batchsize, c, h, w + }); + in.CopyDataFromHostPtr<float>(x, n); + + PReLU prelu; + singa::LayerConf conf; + singa::PReLUConf *preluconf = conf.mutable_prelu_conf(); + preluconf->set_channel_shared(false); + preluconf->set_format("NCHW"); + prelu.Setup(conf); + + const float neg_slope[] = { 0.25f, 0.5f, 0.75f }; + singa::Tensor a(singa::Shape { + c + }); + a.CopyDataFromHostPtr<float>(neg_slope, c); + prelu.Set_a(a); + + singa::Tensor out = prelu.Forward(singa::kTrain, in); + + const float grad[] = { 1.f, 2.f, -2.f, -1.f, -1.f, -3.f, 2.f, -2.f, 1.f, 1.f, + -2.f, 0.f }; + singa::Tensor out_diff(singa::Shape { + batchsize, c, h, w + }); + out_diff.CopyDataFromHostPtr<float>(grad, n); + const auto ret = prelu.Backward(singa::kTrain, out_diff); + const float *xptr = ret.first.data<const float *>(); + const float *aptr = ret.second.at(0).data<const float *>(); + float *dx = new float[n]; + size_t div_factor = prelu.Channel_shared() ? c : 1; + size_t params = prelu.Channel_shared() ? 1 : c; + float da[] = { 0.f, 0.f, 0.f }; + if (prelu.Format() == "NCHW") { + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + dx[i] = grad[i] * + (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f)); + } + for (size_t i = 0; i < n; i++) { + size_t pos = i / (h * w) % c / div_factor; + da[pos] += grad[i] * std::min(x[i], 0.f); + } + } else if (prelu.Format() == "NHWC") { + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + dx[i] = grad[i] * + (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f)); + } + for (size_t i = 0; i < n; i++) { + size_t pos = i % c / div_factor; + da[pos] += grad[i] * std::min(x[i], 0.f); + } + } + for (size_t i = 0; i < n; i++) + EXPECT_FLOAT_EQ(dx[i], xptr[i]); + for (size_t i = 0; i < params; i++) + EXPECT_FLOAT_EQ(da[i], aptr[i]); +}
