Merge pull request #8269 from JiayiFeng/fix_as_numpy

refine error msg and add unit tests for cpp reader

python/paddle/v2/fluid/executor.py

@@ -47,27 +47,13 @@ def as_numpy(tensor):
         return [as_numpy(t) for t in tensor]
     assert isinstance(tensor, core.LoDTensor)
     lod = tensor.lod()
-    tensor_data = np.array(tensor)
-    if len(lod) == 0:
-        ans = tensor_data
-    else:
-        raise RuntimeError("LoD Calculate lacks unit tests and buggy")
-    # elif len(lod) == 1:
-    #     ans = []
-    #     idx = 0
-    #     while idx < len(lod) - 1:
-    #         ans.append(tensor_data[lod[idx]:lod[idx + 1]])
-    #         idx += 1
-    # else:
-    #     for l in reversed(lod):
-    #         ans = []
-    #         idx = 0
-    #         while idx < len(l) - 1:
-    #             ans.append(tensor_data[l[idx]:l[idx + 1]])
-    #             idx += 1
-    #         tensor_data = ans
-    #     ans = tensor_data
-    return ans
+    if len(lod) > 0:
+        raise RuntimeError(
+            "Some of your featched tensors hold LoD information. \
+            They can not be completely cast to Python ndarray. \
+            Please set the parameter 'return_numpy' as 'False' to \
+            return LoDTensor itself directly.")
+    return np.array(tensor)
 
 
 def has_feed_operators(block, feed_targets, feed_holder_name):
@@ -306,7 +292,6 @@ class Executor(object):
             core.get_fetch_variable(scope, fetch_var_name, i)
             for i in xrange(len(fetch_list))
         ]
-
         if return_numpy:
             outs = as_numpy(outs)
         return outs

python/paddle/v2/fluid/tests/test_cpp_reader.py

@@ -32,31 +32,43 @@ create_random_data_generator_op = block.append_op(
         "min": 0.0,
         "max": 1.0
     })
+shuffle_reader = block.create_var(
+    type=fluid.core.VarDesc.VarType.READER, name="ShuffleReader")
+shuffle_reader.desc.set_lod_levels([0, 0])
 
-out1 = block.create_var(
-    type=fluid.core.VarDesc.VarType.LOD_TENSOR,
-    name="Out1",
-    shape=[10, 2],
-    dtype="float32",
-    lod_level=1)
-out2 = block.create_var(
-    type=fluid.core.VarDesc.VarType.LOD_TENSOR,
-    name="Out2",
-    shape=[10, 1],
-    dtype="float32",
-    lod_level=1)
+create_shuffle_reader_op = block.append_op(
+    type="create_shuffle_reader",
+    inputs={"UnderlyingReader": random_reader},
+    outputs={"Out": shuffle_reader},
+    attrs={"buffer_size": 7})
+
+batch_reader = block.create_var(
+    type=fluid.core.VarDesc.VarType.READER, name="BatchReader")
+batch_reader.desc.set_lod_levels([1, 1])
+
+create_batch_reader_op = block.append_op(
+    type="create_batch_reader",
+    inputs={"UnderlyingReader": shuffle_reader},
+    outputs={"Out": batch_reader},
+    attrs={"batch_size": 10})
+
+out1 = block.create_var(type=fluid.core.VarDesc.VarType.LOD_TENSOR, name="Out1")
+out2 = block.create_var(type=fluid.core.VarDesc.VarType.LOD_TENSOR, name="Out2")
 
 read_op = block.append_op(
-    type="read",
-    inputs={"Reader": random_reader},
+    type="read", inputs={"Reader": batch_reader},
     outputs={"Out": [out1, out2]})
 
 place = fluid.CPUPlace()
 exe = fluid.Executor(place)
 
-[res1, res2] = exe.run(prog, fetch_list=[out1, out2])
+[res1, res2] = exe.run(prog, fetch_list=[out1, out2], return_numpy=False)
+
+test_pass = res1.lod() == [range(0, 11)] and res1.lod() == [
+    range(0, 11)
+] and np.array(res1).shape == (10, 2) and np.array(res2).shape == (10, 1)
 
-if len(res1) == 0 or len(res2) == 0:
+if not test_pass:
     exit(1)
 
 exit(0)