Fix ALL RNN error

paddle/operators/while_op.cc

@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
+#include <cmath>
 #include <vector>
 #include "paddle/framework/executor.h"
 #include "paddle/framework/lod_tensor_array.h"
@@ -201,6 +202,15 @@ class WhileGradOp : public framework::OperatorBase {
             VLOG(10) << "Checking " << var_name;
             PADDLE_ENFORCE(!framework::HasNAN(var->Get<framework::LoDTensor>()),
                            "%s has NAN", var_name);
+            if (var->Get<framework::LoDTensor>().type() ==
+                typeid(float)) {  // NOLINT
+              auto &tensor = var->Get<framework::LoDTensor>();
+              auto *buf = tensor.data<float>();
+              for (int64_t i = 0; i < tensor.numel(); ++i) {
+                PADDLE_ENFORCE(!std::isnan(buf[i]));
+              }
+              VLOG(10) << buf[0];
+            }
           }
         };
         check_var_no_nan(cur_scope, inside_grad_name);
@@ -210,7 +220,7 @@ class WhileGradOp : public framework::OperatorBase {
             "sum", {{"X", {pg_names[param_id], new_inside_name}}},
             {{"Out", {pg_names[param_id]}}}, framework::AttributeMap{});
         sum_op->Run(cur_scope, dev_place);
-        check_var_no_nan(cur_scope, pg_names[param_id]);
+        check_var_no_nan(scope, pg_names[param_id]);
         cur_scope.Rename(new_inside_name, inside_grad_name);
       }
     }

paddle/pybind/tensor_py.h

@@ -77,10 +77,10 @@ struct CastToPyBufferImpl<true, I, ARGS...> {
       } else if (paddle::platform::is_cpu_place(tensor.place())) {
         dst_tensor = tensor;
       }
-      return py::buffer_info(
-          dst_tensor.mutable_data<CUR_TYPE>(dst_tensor.place()),
-          sizeof(CUR_TYPE), py::format_descriptor<CUR_TYPE>::format(),
-          (size_t)framework::arity(dst_tensor.dims()), dims_outside, strides);
+      return py::buffer_info(dst_tensor.data<CUR_TYPE>(), sizeof(CUR_TYPE),
+                             py::format_descriptor<CUR_TYPE>::format(),
+                             (size_t)framework::arity(dst_tensor.dims()),
+                             dims_outside, strides);
     } else {
       constexpr bool less = I + 1 < std::tuple_size<std::tuple<ARGS...>>::value;
       return CastToPyBufferImpl<less, I + 1, ARGS...>()(tensor);

python/paddle/v2/fluid/tests/test_dynrnn_gradient_check.py

@@ -294,7 +294,7 @@ class TestSimpleMulWithMemory(unittest.TestCase):
             assert isinstance(Out, Output)
             Out.out(o)
 
-    # @many_times(10)
+    @many_times(10)
     @prog_scope()
     def test_forward_backward(self):
         py_rnn = TestSimpleMulWithMemory.SimpleMulWithMemory()
@@ -330,10 +330,8 @@ class TestSimpleMulWithMemory(unittest.TestCase):
                                         ],
                                         return_numpy=False))
         last_by_py, = py_rnn.exe().values()
-        print w_g[0]
-        self.assertTrue(numpy.allclose(last_np, last_by_py))
         w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME)
-        # print w_g_num[0], w_g[0]
+        self.assertTrue(numpy.allclose(last_np, last_by_py))
 
         self.assertTrue(numpy.allclose(w_g_num, w_g, rtol=0.1))
         i_g_num = py_rnn.get_numeric_gradient_of_input(self.DATA_NAME)