diff --git a/paddle/framework/shape_inference.cc b/paddle/framework/shape_inference.cc
index 0af41b164f5894db17b2f86d4eba371cf05e3b41..2298507471c54c5b7751beff900466737eea36d4 100644
--- a/paddle/framework/shape_inference.cc
+++ b/paddle/framework/shape_inference.cc
@@ -22,6 +22,12 @@ std::vector<framework::DDim> InferShapeContext::GetInputsDim(
   return GetDims(names);
 }
 
+DDim InferShapeContext::GetInputsElementDim(const std::string &name,
+                                            int idx) const {
+  const std::vector<std::string> &names = Inputs(name);
+  return this->GetDim(names[idx]);
+}
+
 void InferShapeContext::SetOutputsDim(
     const std::string &name, const std::vector<framework::DDim> &dims) {
   auto &names = Outputs(name);
diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h
index 05dc47f06ac81f0acb6d0317cbecb3009c7dd7f0..46f2ea84b4b64292cc9026ef9864621efba79c7a 100644
--- a/paddle/framework/shape_inference.h
+++ b/paddle/framework/shape_inference.h
@@ -37,6 +37,7 @@ class InferShapeContext {
   virtual framework::DDim GetInputDim(const std::string &name) const = 0;
 
   std::vector<framework::DDim> GetInputsDim(const std::string &name) const;
+  DDim GetInputsElementDim(const std::string &name, int idx) const;
 
   virtual void SetOutputDim(const std::string &name, const DDim &dim) = 0;
   void SetOutputsDim(const std::string &name,
diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc
index 68b4f7705995e5ecb6c9b8216db7373c1777a31e..59460f6c879cf2c14fd27e33ecb1ba45b21da485 100644
--- a/paddle/operators/while_op.cc
+++ b/paddle/operators/while_op.cc
@@ -287,7 +287,6 @@ class WhileGradOpShapeInference : public framework::InferShapeBase {
 
     auto p_names = ctx->Inputs(kParameters);
     auto pg_names = ctx->Outputs(kParamGrads);
-    auto dims = ctx->GetInputsDim(kParameters);
     auto var_types = ctx->GetInputsVarType(kParameters);
     std::vector<std::string> names_to_set;
     std::vector<framework::DDim> dims_to_set;
@@ -295,13 +294,14 @@ class WhileGradOpShapeInference : public framework::InferShapeBase {
       if (pg_names[i] == framework::kEmptyVarName) {
         continue;
       }
+      auto dims = ctx->GetInputsElementDim(kParameters, i);
       if (var_types[i] == framework::VarDesc::LOD_TENSOR) {
         names_to_set.push_back(pg_names[i]);
-        dims_to_set.push_back(dims[i]);
+        dims_to_set.push_back(dims);
       } else if (var_types[i] == framework::VarDesc::LOD_TENSOR_ARRAY) {
         // not sure how to set the dim of LOD_TENSOR_ARRAY
         names_to_set.push_back(pg_names[i]);
-        dims_to_set.push_back(dims[i]);
+        dims_to_set.push_back(dims);
       }
     }
     ctx->SetDims(names_to_set, dims_to_set);