piiswrong closed pull request #10336: Fix MKLDNN sigmoid/softrelu issue
URL: https://github.com/apache/incubator-mxnet/pull/10336
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/operator/nn/activation-inl.h b/src/operator/nn/activation-inl.h
index 89a369c6717..32a7a5ad617 100644
--- a/src/operator/nn/activation-inl.h
+++ b/src/operator/nn/activation-inl.h
@@ -201,7 +201,7 @@ void ActivationGradCompute(const nnvm::NodeAttrs& attrs,
const std::vector<TBlob>& inputs,
const std::vector<OpReqType>& req,
const std::vector<TBlob>& outputs) {
-#if MXNET_USE_CUDNN == 1
+#if (MXNET_USE_CUDNN == 1 || MXNET_USE_MKLDNN == 1)
CHECK_EQ(inputs.size(), 3U);
#else
CHECK_EQ(inputs.size(), 2U);
diff --git a/src/operator/nn/activation.cc b/src/operator/nn/activation.cc
index 89059321b69..08028265c48 100644
--- a/src/operator/nn/activation.cc
+++ b/src/operator/nn/activation.cc
@@ -44,7 +44,7 @@ struct ActivationGrad {
const std::vector<nnvm::NodeEntry>&
ograds) const {
std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
heads.emplace_back(nnvm::NodeEntry{n, activation::kOut, 0});
-#if MXNET_USE_CUDNN == 1
+#if (MXNET_USE_CUDNN == 1 || MXNET_USE_MKLDNN == 1)
heads.push_back(n->inputs[activation::kData]);
#endif
return MakeGradNode(op_name, n, heads, n->attrs.dict);
@@ -74,15 +74,11 @@ void ActivationGradComputeExCPU(const nnvm::NodeAttrs&
attrs,
const std::vector<NDArray>& inputs,
const std::vector<OpReqType>& req,
const std::vector<NDArray>& outputs) {
-#if MXNET_USE_CUDNN == 1
CHECK_EQ(inputs.size(), 3U);
-#else
- CHECK_EQ(inputs.size(), 2U);
-#endif
const ActivationParam& param = nnvm::get<ActivationParam>(attrs.parsed);
if (SupportMKLDNN(inputs[0])) {
MKLDNN_OPCHECK_INIT(true, outputs.size(), inputs, outputs);
- MKLDNNActivationBackward(attrs, ctx, inputs[0], inputs[1], req[0],
+ MKLDNNActivationBackward(attrs, ctx, inputs[0], inputs[2], req[0],
outputs[0]);
MKLDNN_OPCHECK_RUN(ActivationGradCompute<cpu>, attrs, ctx, inputs, req,
outputs);
return;
@@ -116,13 +112,13 @@ inline static bool BackwardActStorageType(const
nnvm::NodeAttrs& attrs,
DispatchMode* dispatch_mode,
std::vector<int> *in_attrs,
std::vector<int> *out_attrs) {
-#if MXNET_USE_CUDNN == 1
+#if (MXNET_USE_CUDNN == 1 || MXNET_USE_MKLDNN == 1)
CHECK_EQ(in_attrs->size(), 3U);
#else
CHECK_EQ(in_attrs->size(), 2U);
#endif
CHECK_EQ(out_attrs->size(), 1U);
-#if MXNET_USE_CUDNN == 1
+#if (MXNET_USE_CUDNN == 1 || MXNET_USE_MKLDNN == 1)
bool ret = ElemwiseStorageType<3, 1, false, false, false>(attrs, dev_mask,
dispatch_mode,
in_attrs,
out_attrs);
diff --git a/src/operator/nn/mkldnn/mkldnn_act.cc
b/src/operator/nn/mkldnn/mkldnn_act.cc
index 8c19850ced3..9be5bfbc150 100644
--- a/src/operator/nn/mkldnn/mkldnn_act.cc
+++ b/src/operator/nn/mkldnn/mkldnn_act.cc
@@ -43,13 +43,10 @@ namespace mxnet {
namespace op {
bool SupportMKLDNNAct(const ActivationParam& param) {
- // We only enable ReLU for now. It seems other activations have some
precision
- // problems.
- return param.act_type == activation::kReLU;
-#if 0
+ return param.act_type == activation::kReLU
|| param.act_type == activation::kSigmoid
- || param.act_type == activation::kSoftReLU;
-#endif
+ || param.act_type == activation::kSoftReLU
+ || param.act_type == activation::kTanh;
}
static inline mkldnn::algorithm GetMKLDNNActAlgo(const ActivationParam& param)
{
diff --git a/tests/python/unittest/test_gluon.py
b/tests/python/unittest/test_gluon.py
index 952fdf7e366..d91b3f02cd3 100644
--- a/tests/python/unittest/test_gluon.py
+++ b/tests/python/unittest/test_gluon.py
@@ -717,8 +717,8 @@ def test_lambda():
input_data = mx.nd.random.uniform(shape=(2, 3, 5, 7))
out1, out2, out3 = net1(input_data), net2(input_data), net3(input_data)
- assert_almost_equal(out1.asnumpy(), out2.asnumpy())
- assert_almost_equal(out1.asnumpy(), out3.asnumpy())
+ assert_almost_equal(out1.asnumpy(), out2.asnumpy(), rtol=1e-3)
+ assert_almost_equal(out1.asnumpy(), out3.asnumpy(), rtol=1e-3)
@with_seed()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
