From d8fe517bf80a7cbf94885dbf6f527649490e1693 Mon Sep 17 00:00:00 2001 From: Huihuang Zheng Date: Thu, 16 Jul 2020 10:22:08 +0800 Subject: [PATCH] Add Support for SelectedRows for Transpose OP and Fix a Bug That SelectedRows Cannot be Supported in SimNet (#25536) This PR fixes a bug that SelectedRows cannot be supported in SimNet. The reason of this bug is that dygraph basic_engine didn't copy var's type when the var needs to be accumulated during backward. So when a var is SelectedRows and needs to be accumulated, like SimNet which calls net for two times, the var's type will be changed to default LoDTensor thus bug happens. To fix it, we just also copy the type. Without this PR, the accumulated SelectedRows parameters in dygraph will be changed into LoDTensor. So when we fixed the bug of supporting SelectedRows in SimNet, we found `test_imperative_lod_tensor_to_selected_rows` failed and threw the error that SelectedRows was not supported for Transpose OP. To fix it, too, this PR also added support for SelectedRows for Transpose OP. --- paddle/fluid/imperative/basic_engine.cc | 4 +- paddle/fluid/operators/transpose_op.cu | 48 ++++++++++++------- paddle/fluid/operators/transpose_op.h | 42 ++++++++++------ .../dygraph_to_static/simnet_dygraph_model.py | 2 +- .../dygraph_to_static/test_simnet.py | 1 - ..._imperative_lod_tensor_to_selected_rows.py | 3 +- 6 files changed, 63 insertions(+), 37 deletions(-) diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc index 60bc88ca723..6c2f74e2712 100644 --- a/paddle/fluid/imperative/basic_engine.cc +++ b/paddle/fluid/imperative/basic_engine.cc @@ -205,7 +205,9 @@ void BasicEngine::Execute() { continue; } - var = std::make_shared(var->Name()); + auto tmp_var = std::make_shared(var->Name()); + tmp_var->SetType(var->Type()); + var = tmp_var; need_accu_var_list_.emplace_back(iter->second.get(), var); } } diff --git a/paddle/fluid/operators/transpose_op.cu b/paddle/fluid/operators/transpose_op.cu index e9e55c20fc5..79dd29ebc69 100644 --- a/paddle/fluid/operators/transpose_op.cu +++ b/paddle/fluid/operators/transpose_op.cu @@ -660,19 +660,26 @@ template class TransposeGPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* x = context.Input("X"); - auto* out = context.Output("Out"); - out->mutable_data(context.GetPlace()); - if (out->numel() == 0) { + auto* x = context.InputVar("X"); + auto* out = context.OutputVar("Out"); + + const framework::Tensor* x_tensor = + GetLoDTensorOrSelectedRowsValueFromVar(*x); + framework::Tensor* out_tensor = + GetMutableLoDTensorOrSelectedRowsValueFromVar(out); + + out_tensor->mutable_data(context.GetPlace()); + if (out_tensor->numel() == 0) { return; } std::vector axis = context.Attr>("axis"); int ndims = axis.size(); const auto& dev_ctx = context.template device_context(); - auto ret = TransposeSimple::run(dev_ctx, *x, axis, out); + auto ret = TransposeSimple::run(dev_ctx, *x_tensor, axis, out_tensor); if (!ret) { - TransCompute(ndims, dev_ctx, *x, out, axis); + TransCompute(ndims, dev_ctx, *x_tensor, out_tensor, + axis); } } }; @@ -680,14 +687,19 @@ template class TransposeGradGPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* out_grad = - context.Input(framework::GradVarName("Out")); - auto* x_grad = - context.Output(framework::GradVarName("X")); - if (!x_grad) return; - - x_grad->mutable_data(context.GetPlace()); - if (x_grad->numel() == 0) { + auto* out_grad = context.InputVar(framework::GradVarName("Out")); + auto* x_grad = context.OutputVar(framework::GradVarName("X")); + if (!x_grad) { + return; + } + + const framework::Tensor* out_grad_tensor = + GetLoDTensorOrSelectedRowsValueFromVar(*out_grad); + framework::Tensor* x_grad_tensor = + GetMutableLoDTensorOrSelectedRowsValueFromVar(x_grad); + + x_grad_tensor->mutable_data(context.GetPlace()); + if (x_grad_tensor->numel() == 0) { return; } std::vector axis = context.Attr>("axis"); @@ -699,11 +711,11 @@ class TransposeGradGPUKernel : public framework::OpKernel { int ndims = axis.size(); const auto& dev_ctx = context.template device_context(); - auto ret = - TransposeSimple::run(dev_ctx, *out_grad, reversed_axis, x_grad); + auto ret = TransposeSimple::run(dev_ctx, *out_grad_tensor, reversed_axis, + x_grad_tensor); if (!ret) { - TransCompute(ndims, dev_ctx, *out_grad, x_grad, - reversed_axis); + TransCompute(ndims, dev_ctx, *out_grad_tensor, + x_grad_tensor, reversed_axis); } } }; diff --git a/paddle/fluid/operators/transpose_op.h b/paddle/fluid/operators/transpose_op.h index f2951e90ebe..d7f5c3dd457 100644 --- a/paddle/fluid/operators/transpose_op.h +++ b/paddle/fluid/operators/transpose_op.h @@ -64,16 +64,23 @@ template class TransposeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* x = context.Input("X"); - auto* out = context.Output("Out"); - out->mutable_data(context.GetPlace()); - if (out->numel() == 0) { + auto* x = context.InputVar("X"); + auto* out = context.OutputVar("Out"); + + const framework::Tensor* x_tensor = + GetLoDTensorOrSelectedRowsValueFromVar(*x); + framework::Tensor* out_tensor = + GetMutableLoDTensorOrSelectedRowsValueFromVar(out); + + out_tensor->mutable_data(context.GetPlace()); + if (out_tensor->numel() == 0) { return; } + std::vector axis = context.Attr>("axis"); int ndims = axis.size(); auto& dev_ctx = context.template device_context(); - TransCompute(ndims, dev_ctx, *x, out, axis); + TransCompute(ndims, dev_ctx, *x_tensor, out_tensor, axis); } }; @@ -81,14 +88,19 @@ template class TransposeGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* out_grad = - context.Input(framework::GradVarName("Out")); - auto* x_grad = - context.Output(framework::GradVarName("X")); - if (!x_grad) return; - - x_grad->mutable_data(context.GetPlace()); - if (x_grad->numel() == 0) { + auto* out_grad = context.InputVar(framework::GradVarName("Out")); + auto* x_grad = context.OutputVar(framework::GradVarName("X")); + + if (!x_grad) { + return; + } + const framework::Tensor* out_grad_tensor = + GetLoDTensorOrSelectedRowsValueFromVar(*out_grad); + framework::Tensor* x_grad_tensor = + GetMutableLoDTensorOrSelectedRowsValueFromVar(x_grad); + + x_grad_tensor->mutable_data(context.GetPlace()); + if (x_grad_tensor->numel() == 0) { return; } @@ -101,8 +113,8 @@ class TransposeGradKernel : public framework::OpKernel { int ndims = axis.size(); auto& dev_ctx = context.template device_context(); - TransCompute(ndims, dev_ctx, *out_grad, x_grad, - reversed_axis); + TransCompute(ndims, dev_ctx, *out_grad_tensor, + x_grad_tensor, reversed_axis); } }; diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py index 2520b372288..331eeeb3de6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py @@ -42,7 +42,7 @@ class EmbeddingLayer(object): # causes crush in dy2stat. Set it to True after fixing it. emb = Embedding( size=[self.dict_size, self.emb_dim], - is_sparse=False, + is_sparse=True, padding_idx=self.padding_idx, param_attr=attr.ParamAttr( name=self.name, initializer=fluid.initializer.Xavier())) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py index 94b9bb86be2..373e942f6f3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py @@ -149,7 +149,6 @@ def train(conf_dict, to_static): pred = pos_score _, neg_score = net(left, neg_right) avg_cost = loss.compute(pos_score, neg_score) - #avg_cost = loss.compute(pos_score, pos_score) losses.append(np.mean(avg_cost.numpy())) avg_cost.backward() optimizer.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py index 477b3be76f3..69fd7d80327 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py @@ -186,7 +186,8 @@ class TestDygraphSimpleNet(unittest.TestCase): k - 1]] = out[k] self.assertTrue( - np.array_equal(static_loss_value, dy_loss_value)) + np.allclose( + static_loss_value, dy_loss_value, rtol=1e-3)) for key, value in six.iteritems(static_param_init): self.assertTrue(np.array_equal(value, dy_param_init[key])) for key, value in six.iteritems(static_param_updated): -- GitLab