From c7026d034c533156a7efbb907770c0be71cd32a7 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sat, 29 Jun 2019 13:34:47 +0530 Subject: [PATCH 1/4] add higher order support for sinh cosh --- src/operator/tensor/elemwise_unary_op_trig.cc | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/operator/tensor/elemwise_unary_op_trig.cc b/src/operator/tensor/elemwise_unary_op_trig.cc index b7cf76e4eb2d..115ba7ff02e4 100644 --- a/src/operator/tensor/elemwise_unary_op_trig.cc +++ b/src/operator/tensor/elemwise_unary_op_trig.cc @@ -257,7 +257,30 @@ The storage type of ``sinh`` output depends upon the input storage type: )code" ADD_FILELINE) .set_attr("FGradient", ElemwiseGradUseIn{ "_backward_sinh" }); -MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU_DR(_backward_sinh, unary_bwd); +MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU_DR(_backward_sinh, unary_bwd) +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + // ograds[0]: d^2L/dx^2 + // inputs[0]: dL/dy + // inputs[1]: x (ElemwiseUseIn) + // f(x) = sinh(x) + // f'(x) = cosh(x) + // f''(x) = sinh(x) + auto dydx = MakeNode("cosh", n->attrs.name + "_dydx", + {n->inputs[1]}, nullptr, &n); + auto d2ydx2 = MakeNode("sinh", n->attrs.name + "_grad_grad_mid", {n->inputs[1]}, nullptr, &n); + + auto grad_grad_mid = MakeNode("elemwise_mul", n->attrs.name + "backward_grad_grad_mid", + {n->inputs[0], nnvm::NodeEntry{d2ydx2}}, nullptr, &n); + + std::vector ret; + + ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name + "_backward_grad_grad", + {ograds[0], nnvm::NodeEntry{dydx}}, nullptr, &n)); + ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name + "_backward_grad_grad_in", + {ograds[0], nnvm::NodeEntry{grad_grad_mid}}, nullptr, &n)); + return ret; + }); // cosh MXNET_OPERATOR_REGISTER_UNARY_WITH_SPARSE_DR(cosh, cpu, mshadow_op::cosh) @@ -272,7 +295,31 @@ The storage type of ``cosh`` output is always dense )code" ADD_FILELINE) .set_attr("FGradient", ElemwiseGradUseIn{ "_backward_cosh" }); -MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU(_backward_cosh, unary_bwd); +MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU(_backward_cosh, unary_bwd) +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + // ograds[0]: d^2L/dx^2 + // inputs[0]: dL/dy + // inputs[1]: x (ElemwiseUseIn) + // f(x) = cosh(x) + // f'(x) = sinh(x) + // f''(x) = cosh(x) + auto dydx = MakeNode("sinh", n->attrs.name + "_dydx", + {n->inputs[1]}, nullptr, &n); + auto d2ydx2 = MakeNode("cosh", n->attrs.name + "_grad_grad_mid", {n->inputs[1]}, nullptr, &n); + + auto grad_grad_mid = MakeNode("elemwise_mul", n->attrs.name + "backward_grad_grad_mid", + {n->inputs[0], nnvm::NodeEntry{d2ydx2}}, nullptr, &n); + + std::vector ret; + + ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name + "_backward_grad_grad", + {ograds[0], nnvm::NodeEntry{dydx}}, nullptr, &n)); + ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name + "_backward_grad_grad_in", + {ograds[0], nnvm::NodeEntry{grad_grad_mid}}, nullptr, &n)); + return ret; + }); + // tanh MXNET_OPERATOR_REGISTER_UNARY_WITH_RSP_CSR(tanh, cpu, mshadow_op::tanh) From 093e854ecf20d20650431de4fed9403a7d82b240 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sat, 29 Jun 2019 13:35:04 +0530 Subject: [PATCH 2/4] add relevant tests --- .../python/unittest/test_higher_order_grad.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 4f1ea9a6c7b8..a697aa210d5d 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -106,6 +106,34 @@ def grad_grad_op(x): check_second_order_unary(array, log10, grad_grad_op) +@with_seed() +def test_sinh(): + def sinh(x): + return nd.sinh(x) + + def grad_grad_op(x): + return sinh(x) + + for dim in range(1, 5): + shape = rand_shape_nd(dim) + array = random_arrays(shape) + check_second_order_unary(array, sinh, grad_grad_op) + + +@with_seed() +def test_cosh(): + def cosh(x): + return nd.cosh(x) + + def grad_grad_op(x): + return cosh(x) + + for dim in range(1, 5): + shape = rand_shape_nd(dim) + array = random_arrays(shape) + check_second_order_unary(array, cosh, grad_grad_op) + + def check_second_order_unary(x, op, grad_grad_op): x = nd.array(x) grad_grad_x = grad_grad_op(x) From 794eefdc812c5fab23ee42baed2880b4605e13c0 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Fri, 26 Jul 2019 20:26:36 +0530 Subject: [PATCH 3/4] update comments --- src/operator/tensor/elemwise_unary_op_trig.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/tensor/elemwise_unary_op_trig.cc b/src/operator/tensor/elemwise_unary_op_trig.cc index 8f7fd9b1cabc..8ab8bd597ed8 100644 --- a/src/operator/tensor/elemwise_unary_op_trig.cc +++ b/src/operator/tensor/elemwise_unary_op_trig.cc @@ -260,7 +260,7 @@ The storage type of ``sinh`` output depends upon the input storage type: MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU_DR(_backward_sinh, unary_bwd) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { - // ograds[0]: d^2L/dx^2 + // ograds[0]: head_grad_grads (dL/dxgrad) // inputs[0]: dL/dy // inputs[1]: x (ElemwiseUseIn) // f(x) = sinh(x) @@ -298,7 +298,7 @@ The storage type of ``cosh`` output is always dense MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU(_backward_cosh, unary_bwd) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { - // ograds[0]: d^2L/dx^2 + // ograds[0]: head_grad_grads (dL/dxgrad) // inputs[0]: dL/dy // inputs[1]: x (ElemwiseUseIn) // f(x) = cosh(x) From 283503c869508bb9030f7b3192b05f5fdf48cf3e Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Wed, 11 Sep 2019 19:08:26 +0530 Subject: [PATCH 4/4] retrigger CI