apeforest commented on a change in pull request #15480: [MXNET-978] Higher
Order Gradient Support `elemwise_mul`, `elemwise_add`.
URL: https://github.com/apache/incubator-mxnet/pull/15480#discussion_r300884866
##########
File path: src/operator/tensor/elemwise_binary_op_basic.cc
##########
@@ -274,7 +298,43 @@ NNVM_REGISTER_OP(_backward_div)
.set_attr<FCompute>("FCompute<cpu>", ElemwiseBinaryOp::BackwardUseIn<
cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>)
.set_attr<FComputeEx>("FComputeEx<cpu>", ElemwiseBinaryOp::BackwardUseInEx<
- cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>);
+ cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>)
+.set_attr<nnvm::FGradient>("FGradient",
+ [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
+ // z = x / y
+ // NodeEntry{n, 0, 0} : z_grad * (1/y)
+ // NodeEntry{n, 1, 0} : z_grad * (-x/y^2)
+ // n->inputs[0] : z_grad
+ // n->inputs[1] : x
+ // n->inputs[1] : y
+ // ograds[0] : head_grads
+ // f(x, y) = x / y
+ // dx = z_grad * (1/y), dy = z_grad * (-x/y^2)
+ // d2x = 0, d2y = dy * (-2/x) = (2x/y^3), dz_grad = (dx + dy) / (z_grad)
+ auto dx = nnvm::NodeEntry{n, 0, 0};
+ auto dy = nnvm::NodeEntry{n, 1, 0};
+ auto dx_add_dy = MakeNode("elemwise_add", n->attrs.name + "_x_add_y",
+ {dx, dy}, nullptr, &n);
+ auto dz_grad = MakeNode("elemwise_div", n->attrs.name + "_x_add_y",
+ {nnvm::NodeEntry{dx_add_dy},
n->inputs[0]}, nullptr, &n);
+
+ const std::unordered_map<std::string, std::string> two = {{"scalar",
"-2.0"}};
+ auto y = n->inputs[2];
+ auto r_y = MakeNode("reciprocal", n->attrs.name + "_r_y", {y}, nullptr,
&n);
+ auto neg_two_r_y = MakeNode("_mul_scalar", n->attrs.name + "_neg_two_r_y",
+ {nnvm::NodeEntry{r_y}}, &two, &n);
+ auto d2y = MakeNode("elemwise_mul", n->attrs.name + "_d2y",
+ {nnvm::NodeEntry{neg_two_r_y}, dy}, &two, &n);
+
+ std::vector<nnvm::NodeEntry> ret;
+ ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name +
"_backward_grad_grad",
+ {ograds[0], nnvm::NodeEntry{dz_grad}}, nullptr,
&n));
+ ret.emplace_back(MakeNode("zeros_like", n->attrs.name +
"_backward_grad_grad_x",
Review comment:
Same here. Please test z = x / (x^2 + 1)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
