diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h
index 568f4e89819c8345d8908634f6fa56f09483a763..fac5cd20aa7f9db0792f8102bb442192ab1ad63f 100644
--- a/paddle/framework/lod_tensor.h
+++ b/paddle/framework/lod_tensor.h
@@ -51,18 +51,15 @@ bool operator==(const LoD& a, const LoD& b);
  * LoDTensor (Level of details Tensor)
  * see https://en.wikipedia.org/wiki/Level_of_details for reference.
  */
-class LoDTensor {
+class LoDTensor : public Tensor {
  public:
   LoDTensor() {}
-  LoDTensor(const LoD& lod, Tensor* t) : lod_(lod), tensor_(t) {}
 
-  void set_lod(const LoD& lod) { lod_ = lod; }
-
-  void set_tensor(Tensor* tensor) { tensor_ = tensor; }
+  explicit LoDTensor(const LoD& lod) : lod_(lod) {}
 
-  Tensor& tensor() { return *tensor_; }
+  void set_lod(const LoD& lod) { lod_ = lod; }
 
-  LoD lod() { return lod_; }
+  LoD lod() const { return lod_; }
 
   /*
    * Get a element from LoD.
@@ -104,7 +101,6 @@ class LoDTensor {
 
  private:
   LoD lod_;
-  Tensor* tensor_;  // not owned
 };
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc
index 1da8553134f377f7a4fbe8008d12fe8d4a0e47f4..7915326b27a22e9280e3f09d9bbfc2a58f46aff7 100644
--- a/paddle/framework/lod_tensor_test.cc
+++ b/paddle/framework/lod_tensor_test.cc
@@ -36,69 +36,64 @@ class LoDTensorTester : public ::testing::Test {
 
     ASSERT_EQ(lod.size(), 3UL);
 
-    tensor.Resize({20 /*batch size*/, 128 /*dim*/});
+    lod_tensor_.Resize({20 /*batch size*/, 128 /*dim*/});
     // malloc memory
-    tensor.mutable_data<float>(place);
+    lod_tensor_.mutable_data<float>(place);
 
-    lod_tensor.set_lod(lod);
-    lod_tensor.set_tensor(&tensor);
+    lod_tensor_.set_lod(lod);
   }
 
  protected:
   platform::CPUPlace place;
-  Tensor tensor;
-  LoDTensor lod_tensor;
+  LoDTensor lod_tensor_;
 };
 
-TEST_F(LoDTensorTester, NumLevels) { ASSERT_EQ(lod_tensor.NumLevels(), 3UL); }
+TEST_F(LoDTensorTester, NumLevels) { ASSERT_EQ(lod_tensor_.NumLevels(), 3UL); }
 
 TEST_F(LoDTensorTester, NumElements) {
-  ASSERT_EQ(lod_tensor.NumElements(0), 2UL);
-  ASSERT_EQ(lod_tensor.NumElements(1), 4UL);
-  ASSERT_EQ(lod_tensor.NumElements(2), 8UL);
+  ASSERT_EQ(lod_tensor_.NumElements(0), 2UL);
+  ASSERT_EQ(lod_tensor_.NumElements(1), 4UL);
+  ASSERT_EQ(lod_tensor_.NumElements(2), 8UL);
 }
 
 TEST_F(LoDTensorTester, SliceLevels) {
   // slice 1 level
   for (size_t level = 0; level < 3UL; ++level) {
-    LoDTensor new_lod_tensor = lod_tensor;
+    LoDTensor new_lod_tensor = lod_tensor_;
     new_lod_tensor.SliceLevels(level, level + 1);
     ASSERT_EQ(new_lod_tensor.NumLevels(), 1UL);
-    ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor.NumElements(level));
-    ASSERT_EQ(new_lod_tensor.tensor().data<float>(),
-              lod_tensor.tensor().data<float>());
+    ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level));
+    ASSERT_EQ(new_lod_tensor.data<float>(), lod_tensor_.data<float>());
   }
   // slice 2 level
   for (size_t level = 0; level < 2UL; ++level) {
-    LoDTensor new_lod_tensor = lod_tensor;
+    LoDTensor new_lod_tensor = lod_tensor_;
     new_lod_tensor.SliceLevels(level, level + 2);
     ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL);
-    ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor.NumElements(level));
-    ASSERT_EQ(new_lod_tensor.NumElements(1), lod_tensor.NumElements(level + 1));
-    ASSERT_EQ(new_lod_tensor.tensor().data<float>(),
-              lod_tensor.tensor().data<float>());
+    ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level));
+    ASSERT_EQ(new_lod_tensor.NumElements(1),
+              lod_tensor_.NumElements(level + 1));
+    ASSERT_EQ(new_lod_tensor.data<float>(), lod_tensor_.data<float>());
   }
 }
 
 TEST_F(LoDTensorTester, SliceInLevel) {
   size_t level = 0;
-  LoDTensor new_lod_tensor = lod_tensor;
+  LoDTensor new_lod_tensor = lod_tensor_;
   new_lod_tensor.SliceInLevel(level, 0, 2);
   EXPECT_EQ(new_lod_tensor.NumLevels(), 3UL);
   EXPECT_EQ(new_lod_tensor.NumElements(0), 2UL);
   EXPECT_EQ(new_lod_tensor.NumElements(1), 4UL);
   EXPECT_EQ(new_lod_tensor.NumElements(2), 8UL);
-  ASSERT_EQ(new_lod_tensor.tensor().data<float>(),
-            lod_tensor.tensor().data<float>());
+  ASSERT_EQ(new_lod_tensor.data<float>(), lod_tensor_.data<float>());
 
   level = 1;
-  new_lod_tensor = lod_tensor;
+  new_lod_tensor = lod_tensor_;
   new_lod_tensor.SliceInLevel(level, 0, 2);
   ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL);
   ASSERT_EQ(new_lod_tensor.NumElements(0), 2UL);
   ASSERT_EQ(new_lod_tensor.NumElements(1), 4UL);
-  ASSERT_EQ(new_lod_tensor.tensor().data<float>(),
-            lod_tensor.tensor().data<float>());
+  ASSERT_EQ(new_lod_tensor.data<float>(), lod_tensor_.data<float>());
 }
 
 }  // namespace framework
diff --git a/paddle/framework/lod_tensor_test.cu b/paddle/framework/lod_tensor_test.cu
index 1079a36a2e7b24f6f8a5bcbb296355567305a765..97e69cdb2e5e1e64031c899f5e04020665485ba8 100644
--- a/paddle/framework/lod_tensor_test.cu
+++ b/paddle/framework/lod_tensor_test.cu
@@ -26,18 +26,16 @@ __global__ void test(size_t* a, int size) {
 }
 
 TEST(LoDTensor, LoDInGPU) {
-  paddle::framework::Tensor tensor;
   paddle::framework::LoDTensor lod_tensor;
   paddle::platform::GPUPlace place(0);
 
   paddle::framework::LoD src_lod;
   src_lod.push_back(std::vector<size_t>{0, 2, 4, 6, 8, 10, 12, 14});
 
-  tensor.Resize({14, 16});
-  tensor.mutable_data<float>(place);
+  lod_tensor.Resize({14, 16});
+  lod_tensor.mutable_data<float>(place);
 
   lod_tensor.set_lod(src_lod);
-  lod_tensor.set_tensor(&tensor);
   CHECK_EQ(lod_tensor.lod_element(0, 2), 4);
   CHECK_EQ(lod_tensor.lod_element(0, 4), 8);
 
diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc
index e1e122091f7759b1a68f1f982bc2a35e8241f9f0..c57537be4bf67a8db6a49669ab8d2ed1b1324bdc 100644
--- a/paddle/framework/operator.cc
+++ b/paddle/framework/operator.cc
@@ -186,6 +186,48 @@ void OperatorBase::GenerateTemporaryNames() {
   }
 }
 
+template <>
+const Tensor* InferShapeContext::Input<Tensor>(const std::string& name) const {
+  auto* var = InputVar(name);
+  return var == nullptr ? nullptr : GetTensorFromVar(var);
+}
+
+template <>
+const std::vector<const Tensor*> InferShapeContext::MultiInput<Tensor>(
+    const std::string& name) const {
+  auto names = op().Inputs(name);
+  std::vector<const Tensor*> res;
+  res.reserve(names.size());
+  std::transform(names.begin(), names.end(), std::back_inserter(res),
+                 [&](const std::string& sub_name) {
+                   auto var = scope_.FindVar(sub_name);
+                   return var == nullptr ? nullptr : GetTensorFromVar(var);
+                 });
+  return res;
+}
+
+template <>
+Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
+  auto* var = OutputVar(name);
+  return var == nullptr ? nullptr : const_cast<Tensor*>(GetTensorFromVar(var));
+}
+
+template <>
+std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
+    const std::string& name) const {
+  auto names = op().Outputs(name);
+  std::vector<Tensor*> res;
+  res.reserve(names.size());
+  std::transform(names.begin(), names.end(), std::back_inserter(res),
+                 [&](const std::string& sub_name) {
+                   auto var = scope().FindVar(sub_name);
+                   return var == nullptr
+                              ? nullptr
+                              : const_cast<Tensor*>(GetTensorFromVar(var));
+                 });
+  return res;
+}
+
 void OpProtoAndCheckerMaker::Validate() {
   validated_ = true;
   CheckNoDuplicatedInOutAttrs();
diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h
index 4600b06009bcef7d0774d25b816aac4733f30795..adae7bfc3d7d31b1ed0373f01db4ef80343a08f7 100644
--- a/paddle/framework/operator.h
+++ b/paddle/framework/operator.h
@@ -22,6 +22,7 @@ limitations under the License. */
 #include "op_info.h"
 #include "paddle/framework/attribute.h"
 #include "paddle/framework/framework.pb.h"
+#include "paddle/framework/lod_tensor.h"
 #include "paddle/framework/scope.h"
 #include "paddle/framework/tensor.h"
 #include "paddle/platform/device_context.h"
@@ -326,11 +327,27 @@ class InferShapeContext {
     return res;
   }
 
+  const Tensor* GetTensorFromVar(const Variable* var) const {
+    if (var->IsType<LoDTensor>()) {
+      return &var->Get<LoDTensor>();
+    }
+    PADDLE_ENFORCE(var->IsType<Tensor>(),
+                   "The Input(%s) must be LoDTensor or Tensor.");
+    return &var->Get<Tensor>();
+  }
+
  private:
   const OperatorBase& op_;
   const Scope& scope_;
 };
 
+template <>
+const Tensor* InferShapeContext::Input<Tensor>(const std::string& name) const;
+
+template <>
+const std::vector<const Tensor*> InferShapeContext::MultiInput<Tensor>(
+    const std::string& name) const;
+
 template <typename T>
 struct EigenDeviceConverter;
 
@@ -363,9 +380,37 @@ class ExecutionContext : public InferShapeContext {
     return device_context_;
   }
 
+  // redefine Output function,
+  // use Variable::Get instead of Variable::GetMutable
+  template <typename T>
+  T* Output(const std::string& name) const {
+    auto var = OutputVar(name);
+    return var == nullptr ? nullptr : const_cast<T*>(&var->Get<T>());
+  }
+
+  // redefine MultiOutput function.
+  // use Variable::Get instead of Variable::GetMutable
+  template <typename T>
+  std::vector<T*> MultiOutput(const std::string& name) const {
+    auto names = op().Outputs(name);
+    std::vector<T*> res;
+    res.reserve(names.size());
+    std::transform(
+        names.begin(), names.end(), std::back_inserter(res),
+        [&](const std::string& sub_name) { return Output<T>(sub_name); });
+    return res;
+  }
+
   const platform::DeviceContext* device_context_;
 };
 
+template <>
+Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const;
+
+template <>
+std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
+    const std::string& name) const;
+
 class OpKernel {
  public:
   /**
diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc
index 9ca04d402879b6a955d849a32175194df82b65c8..4a6c6381b0341dd3531aa4c09024530ee67bb4f9 100644
--- a/paddle/operators/accuracy_op.cc
+++ b/paddle/operators/accuracy_op.cc
@@ -34,7 +34,7 @@ class AccuracyOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(inference->dims()[0], label->dims()[0],
                       "inference size must be the same as label size");
 
-    ctx.Output<Tensor>("Accuracy")->Resize({1});
+    ctx.Output<framework::LoDTensor>("Accuracy")->Resize({1});
   }
 };
 
diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc
index 8dbd47cf0dfbc265032a9966343eed5c7bd8692e..b43c09d4f09c7f87cc60290bdd2a99cbe46f0d5c 100644
--- a/paddle/operators/add_op.cc
+++ b/paddle/operators/add_op.cc
@@ -26,7 +26,8 @@ class AddOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(ctx.Input<Tensor>("X")->dims(),
                       ctx.Input<Tensor>("Y")->dims(),
                       "Two input of Add Op's dimension must be same.");
-    ctx.Output<Tensor>("Out")->Resize(ctx.Input<Tensor>("X")->dims());
+    ctx.Output<framework::LoDTensor>("Out")->Resize(
+        ctx.Input<Tensor>("X")->dims());
   }
 };
 
diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc
index 0ebefbab26ec8fdf316f852fbb7f6d9f3bbc48eb..72fd179354a4be76a37e4571da168d844f7ce384 100644
--- a/paddle/operators/concat_op.cc
+++ b/paddle/operators/concat_op.cc
@@ -26,7 +26,7 @@ class ConcatOp : public framework::OperatorWithKernel {
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
     auto ins = ctx.MultiInput<framework::Tensor>("X");
-    auto *out = ctx.Output<framework::Tensor>("Out");
+    auto *out = ctx.Output<framework::LoDTensor>("Out");
     size_t axis = static_cast<size_t>(ctx.Attr<int>("axis"));
     size_t n = ins.size();
 
diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc
index 3d0af97aae79089d10d94cb636775c5c6c092d56..253b17d8a1b88eccc58fc458ae8274d2bbd1c323 100644
--- a/paddle/operators/cos_sim_op.cc
+++ b/paddle/operators/cos_sim_op.cc
@@ -46,9 +46,9 @@ class CosSimOp : public framework::OperatorWithKernel {
                    " just 1 (which will be broadcasted to match Input(X)).");
 
     // resize tensor
-    ctx.Output<Tensor>("Out")->Resize({x_dims[0], 1});
-    ctx.Output<Tensor>("XNorm")->Resize({x_dims[0], 1});
-    ctx.Output<Tensor>("YNorm")->Resize({y_dims[0], 1});
+    ctx.Output<framework::LoDTensor>("Out")->Resize({x_dims[0], 1});
+    ctx.Output<framework::LoDTensor>("XNorm")->Resize({x_dims[0], 1});
+    ctx.Output<framework::LoDTensor>("YNorm")->Resize({y_dims[0], 1});
   }
 };
 
@@ -131,8 +131,10 @@ class CosSimOpGrad : public framework::OperatorWithKernel {
                       "Shape of Input(Out@Grad) must be [X.Dim(0), 1].");
 
     // resize tensor
-    auto *x_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
-    auto *y_grad = ctx.Output<Tensor>(framework::GradVarName("Y"));
+    auto *x_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    auto *y_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Y"));
     if (x_grad) x_grad->Resize(x_dims);
     if (y_grad) y_grad->Resize(y_dims);
   }
diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc
index 1742925545d29df5d7df719faaea3b754680ab61..e37c582adbe5b9e728f683d97cc51063ce80c3a2 100644
--- a/paddle/operators/elementwise_mul_op.cc
+++ b/paddle/operators/elementwise_mul_op.cc
@@ -31,7 +31,7 @@ class ElementWiseMulOp : public framework::OperatorWithKernel {
     auto y_dim = ctx.Input<Tensor>("Y")->dims();
     PADDLE_ENFORCE_GE(x_dim.size(), y_dim.size(),
                       "Rank of first input must >= rank of second input.")
-    ctx.Output<Tensor>("Out")->Resize(x_dim);
+    ctx.Output<framework::LoDTensor>("Out")->Resize(x_dim);
   }
 };
 
@@ -80,8 +80,10 @@ class ElementWiseMulOpGrad : public framework::OperatorWithKernel {
     auto x_dims = ctx.Input<Tensor>("X")->dims();
     auto y_dims = ctx.Input<Tensor>("Y")->dims();
     auto out_dims = ctx.Input<Tensor>(framework::GradVarName("Out"))->dims();
-    auto *x_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
-    auto *y_grad = ctx.Output<Tensor>(framework::GradVarName("Y"));
+    auto *x_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    auto *y_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Y"));
 
     PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(),
                       "Rank of first input must >= rank of second input.")
diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc
index 9d51f6e3a16fe96125599bb440d40237aeb9a028..0c9734892aac216709d380ec66acadf792761b14 100644
--- a/paddle/operators/fill_zeros_like_op.cc
+++ b/paddle/operators/fill_zeros_like_op.cc
@@ -23,7 +23,7 @@ class FillZerosLikeOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    ctx.Output<framework::Tensor>("Dst")->Resize(
+    ctx.Output<framework::LoDTensor>("Dst")->Resize(
         ctx.Input<framework::Tensor>("Src")->dims());
   }
 };
diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc
index 123bed296c462c30bddd3bfbd530098fdbfe4856..8883d6d5fed2d8900364cf713a1e8d8b290ef83e 100644
--- a/paddle/operators/gather_op.cc
+++ b/paddle/operators/gather_op.cc
@@ -28,7 +28,7 @@ class GatherOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_GE(batch_size, 0, "Batch size must be >0");
     framework::DDim output_dims(ctx.Input<Tensor>("X")->dims());
     output_dims[0] = batch_size;
-    ctx.Output<Tensor>("Out")->Resize(output_dims);
+    ctx.Output<framework::LoDTensor>("Out")->Resize(output_dims);
   }
 };
 
@@ -38,7 +38,7 @@ class GatherGradOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    auto X_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
+    auto X_grad = ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
     auto X = ctx.Input<Tensor>("X");
 
     X_grad->Resize(X->dims());
diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc
index 3d76516405960c502a46997108049b2db5cab6bf..25b0776a37488e876b0cc88aa3f2aa68e33fb270 100644
--- a/paddle/operators/gaussian_random_op.cc
+++ b/paddle/operators/gaussian_random_op.cc
@@ -44,7 +44,7 @@ class GaussianRandomOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext& context) const override {
-    auto* tensor = context.Output<framework::Tensor>("Out");
+    auto* tensor = context.Output<framework::LoDTensor>("Out");
     auto dims = Attr<std::vector<int>>("dims");
     std::vector<int64_t> temp;
     temp.reserve(dims.size());
diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc
index 94d40890a765413e88a35a6ad995ca97ac84dcda..b3d15f1ec99813d242c86c99faa7385795eef3b1 100644
--- a/paddle/operators/lookup_table_op.cc
+++ b/paddle/operators/lookup_table_op.cc
@@ -25,7 +25,7 @@ class LookupTableOp : public framework::OperatorWithKernel {
   void InferShape(const framework::InferShapeContext &context) const override {
     auto table_t = context.Input<Tensor>("W");
     auto ids_t = context.Input<Tensor>("Ids");
-    auto output_t = context.Output<Tensor>("Out");
+    auto output_t = context.Output<framework::LoDTensor>("Out");
 
     output_t->Resize({ids_t->dims()[0], table_t->dims()[1]});
   }
@@ -56,7 +56,8 @@ class LookupTableOpGrad : public framework::OperatorWithKernel {
  protected:
   void InferShape(const framework::InferShapeContext &context) const override {
     auto table = context.Input<Tensor>("W");
-    auto d_table = context.Output<Tensor>(framework::GradVarName("W"));
+    auto d_table =
+        context.Output<framework::LoDTensor>(framework::GradVarName("W"));
     d_table->Resize(table->dims());
   }
 };
diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc
index d3d0e55a674587fb04f43f24d0790de4358f035a..3e523d31b682d70825d50c2a57b6e98cbf29dcd3 100644
--- a/paddle/operators/mean_op.cc
+++ b/paddle/operators/mean_op.cc
@@ -25,7 +25,7 @@ class MeanOp : public framework::OperatorWithKernel {
   void InferShape(const framework::InferShapeContext &ctx) const override {
     PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"),
                             "Input of MeanOp must be initialized.");
-    ctx.Output<Tensor>("Out")->Resize({1});
+    ctx.Output<framework::LoDTensor>("Out")->Resize({1});
   }
 };
 
@@ -45,7 +45,7 @@ class MeanGradOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    ctx.Output<Tensor>(framework::GradVarName("X"))
+    ctx.Output<framework::LoDTensor>(framework::GradVarName("X"))
         ->Resize(ctx.Input<Tensor>("X")->dims());
   }
 };
diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc
index a4876feb2edf77bd422fa2a7687b0fa7d55dae47..8a583f24edf40ce805ca216ab014bd169f9236df 100644
--- a/paddle/operators/minus_op.cc
+++ b/paddle/operators/minus_op.cc
@@ -33,7 +33,7 @@ class MinusOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(
         left_tensor->numel(), right_tensor->numel(),
         "Minus operator must take two tensor with same num of elements");
-    ctx.Output<framework::Tensor>("Out")->Resize(left_tensor->dims());
+    ctx.Output<framework::LoDTensor>("Out")->Resize(left_tensor->dims());
   }
 };
 
diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc
index 710a56a0e8e2d17162d7d000df226f1537104eb9..015e13de9a09bcd1931ccf91413b6a3f484f82bb 100644
--- a/paddle/operators/mul_op.cc
+++ b/paddle/operators/mul_op.cc
@@ -18,6 +18,7 @@ namespace paddle {
 namespace operators {
 
 using framework::Tensor;
+using framework::LoDTensor;
 
 class MulOp : public framework::OperatorWithKernel {
  public:
@@ -45,7 +46,8 @@ class MulOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(
         x_mat_dims[1], y_mat_dims[0],
         "First matrix's width must be equal with second matrix's height.");
-    ctx.Output<Tensor>("Out")->Resize({x_mat_dims[0], y_mat_dims[1]});
+    ctx.Output<framework::LoDTensor>("Out")->Resize(
+        {x_mat_dims[0], y_mat_dims[1]});
   }
 };
 
@@ -94,8 +96,10 @@ class MulOpGrad : public framework::OperatorWithKernel {
     auto x_dims = ctx.Input<Tensor>("X")->dims();
     auto y_dims = ctx.Input<Tensor>("Y")->dims();
     auto out_dims = ctx.Input<Tensor>(framework::GradVarName("Out"))->dims();
-    auto *x_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
-    auto *y_grad = ctx.Output<Tensor>(framework::GradVarName("Y"));
+    auto *x_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    auto *y_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Y"));
 
     auto x_mat_dims =
         framework::flatten_to_2d(x_dims, Attr<int>("x_num_col_dims"));
diff --git a/paddle/operators/onehot_cross_entropy_op.cc b/paddle/operators/onehot_cross_entropy_op.cc
index 29889926490cbe8448c8ddeecec51832af16c89e..a9baada1cd4cd3af793bf1b0af7b029417e62b08 100644
--- a/paddle/operators/onehot_cross_entropy_op.cc
+++ b/paddle/operators/onehot_cross_entropy_op.cc
@@ -29,7 +29,7 @@ class OnehotCrossEntropyOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(X->dims().size(), 2, "X's dimension must be 2.");
     PADDLE_ENFORCE_EQ(label->dims().size(), 1, "label's dimension must be 1.");
     PADDLE_ENFORCE_EQ(X->dims()[0], label->dims()[0]);
-    ctx.Output<Tensor>("Y")->Resize({X->dims()[0], 1});
+    ctx.Output<framework::LoDTensor>("Y")->Resize({X->dims()[0], 1});
   }
 };
 
@@ -39,7 +39,7 @@ class OnehotCrossEntropyGradientOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    auto dX = ctx.Output<Tensor>(framework::GradVarName("X"));
+    auto dX = ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
     auto X = ctx.Input<Tensor>("X");
 
     dX->Resize(X->dims());
diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc
index 7e78b6ec133981494a65b5e16316ae8fdbd61a60..6cf7bd6f35b7592c41983efd75c1628043070687 100644
--- a/paddle/operators/pad_op.cc
+++ b/paddle/operators/pad_op.cc
@@ -34,7 +34,8 @@ class PadOp : public framework::OperatorWithKernel {
     for (int i = 0; i < x_dim.size(); ++i) {
       out_dims[i] = x_dim[i] + paddings[i * 2] + paddings[i * 2 + 1];
     }
-    ctx.Output<Tensor>("Out")->Resize(framework::make_ddim(out_dims));
+    ctx.Output<framework::LoDTensor>("Out")->Resize(
+        framework::make_ddim(out_dims));
   }
 };
 
@@ -95,9 +96,9 @@ class PadOpGrad : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")),
                             "Input(Out@GRAD) should not be null");
     auto x_dims = ctx.Input<Tensor>("X")->dims();
-    auto *x_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
-    if (x_grad != nullptr) {
-      x_grad->Resize(x_dims);
+    auto *x_g = ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    if (x_g != nullptr) {
+      x_g->Resize(x_dims);
     }
   }
 };
diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc
index e826703c60ca82e1fe690eb78c3d4f92981ef3a2..d3413d7cb9305732e9ddf3cb1bc267f7203097f3 100644
--- a/paddle/operators/recurrent_op.cc
+++ b/paddle/operators/recurrent_op.cc
@@ -26,10 +26,11 @@ namespace operators {
 using Scope = framework::Scope;
 using Variable = framework::Variable;
 using Tensor = framework::Tensor;
+using LoDTensor = framework::LoDTensor;
 
 void RecurrentAlgorithm::InferShape(const Scope& scope) const {
   seq_len_ = scope.FindVar((arg_->inlinks[0]).external)
-                 ->GetMutable<Tensor>()
+                 ->GetMutable<LoDTensor>()
                  ->dims()[0];
   CreateScopes(scope);
   auto step_scopes = GetStepScopes(scope);
@@ -88,7 +89,7 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope) const {
         // the weight are located in parent scope
         for (auto& var_name : input.second) {
           if (!step_scope.FindVar(var_name)) {
-            step_scope.NewVar(var_name)->GetMutable<Tensor>();
+            step_scope.NewVar(var_name)->GetMutable<LoDTensor>();
           }
         }
       }
@@ -106,11 +107,12 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope) const {
 void RecurrentAlgorithm::InitMemories(Scope* step_scope,
                                       bool infer_shape_mode) const {
   for (auto& attr : arg_->memories) {
-    Tensor* pre_mem = step_scope->NewVar(attr.pre_var)->GetMutable<Tensor>();
+    auto* pre_mem = step_scope->NewVar(attr.pre_var)->GetMutable<LoDTensor>();
     PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr,
                    "memory [%s]'s boot variable [%s] not exists", attr.var,
                    attr.boot_var);
-    Tensor* boot_mem = step_scope->FindVar(attr.boot_var)->GetMutable<Tensor>();
+    auto* boot_mem =
+        step_scope->FindVar(attr.boot_var)->GetMutable<LoDTensor>();
     if (infer_shape_mode) {
       pre_mem->Resize(boot_mem->dims());
       PADDLE_ENFORCE_EQ(pre_mem->dims().size(), 2);
@@ -192,9 +194,9 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients(
                    "memory variable [%s] does not exists", attr.var);
     PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr,
                    "boot variable [%s] does not exists", attr.boot_var);
-    Tensor* mem_grad = step_scope->NewVar(attr.var)->GetMutable<Tensor>();
-    Tensor* boot_mem_grad =
-        step_scope->NewVar(attr.boot_var)->GetMutable<Tensor>();
+    auto* mem_grad = step_scope->NewVar(attr.var)->GetMutable<LoDTensor>();
+    auto* boot_mem_grad =
+        step_scope->NewVar(attr.boot_var)->GetMutable<LoDTensor>();
     if (infer_shape_mode) {
       boot_mem_grad->Resize(mem_grad->dims());
     } else {
@@ -205,7 +207,7 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients(
 
 void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const {
   seq_len_ = scope.FindVar((arg_->inlinks[0]).external)
-                 ->GetMutable<Tensor>()
+                 ->GetMutable<LoDTensor>()
                  ->dims()[0];
   auto step_scopes = GetStepScopes(scope);
   rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_,
diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc
index b7061153d2bf13982f14f233e87a87daeeebf5fd..d2817020921dfd3c044922de6a0f2ae0307936bd 100644
--- a/paddle/operators/reshape_op.cc
+++ b/paddle/operators/reshape_op.cc
@@ -46,7 +46,7 @@ class ReshapeOp : public framework::OperatorWithKernel {
     std::transform(shape.begin(), shape.end(), shape_int64.begin(),
                    [](int a) { return static_cast<int64_t>(a); });
     auto out_dims = framework::make_ddim(shape_int64);
-    ctx.Output<framework::Tensor>("Out")->Resize(out_dims);
+    ctx.Output<framework::LoDTensor>("Out")->Resize(out_dims);
   }
 };
 
@@ -90,7 +90,7 @@ class ReshapeGradOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")),
                             "Input(Out@GRAD) shouldn't be null.");
     auto dims = ctx.Input<framework::Tensor>("X")->dims();
-    auto *d_in = ctx.Output<framework::Tensor>(framework::GradVarName("X"));
+    auto *d_in = ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
     d_in->Resize(dims);
   }
 };
diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc
index 97872c67ac99fbf6c9c177d52f1d4069163e8548..6c082cb1825e04accb09019fef28eb2ec6523a5b 100644
--- a/paddle/operators/rnn/recurrent_op_utils.cc
+++ b/paddle/operators/rnn/recurrent_op_utils.cc
@@ -21,6 +21,7 @@ namespace rnn {
 namespace f = paddle::framework;
 
 using Tensor = framework::Tensor;
+using LoDTensor = framework::LoDTensor;
 
 void SegmentInputs(const std::vector<Scope*>& step_scopes,
                    const std::vector<Link>& inlinks, const size_t seq_len,
@@ -31,7 +32,7 @@ void SegmentInputs(const std::vector<Scope*>& step_scopes,
     PADDLE_ENFORCE(input_var != nullptr, "input link [%s] is not in scope.",
                    inlinks[i].external);
 
-    Tensor* input = input_var->GetMutable<Tensor>();
+    LoDTensor* input = input_var->GetMutable<LoDTensor>();
     f::DDim dims = input->dims();
     PADDLE_ENFORCE(static_cast<size_t>(dims[0]) == seq_len,
                    "all the inlinks must have same length");
@@ -40,6 +41,8 @@ void SegmentInputs(const std::vector<Scope*>& step_scopes,
       Tensor* step_input =
           step_scopes[j]->NewVar(inlinks[i].internal)->GetMutable<Tensor>();
       if (!infer_shape_mode) {
+        // The input of operators of each step is Tensor here.
+        // Maybe need to modify Slice function.
         *step_input = input->Slice<float>(j, j + 1);
       }
       step_input->Resize(step_dims);
@@ -54,21 +57,23 @@ void ConcatOutputs(const std::vector<Scope*>& step_scopes,
     auto output_var = step_scopes[0]->FindVar(outlinks[i].external);
     PADDLE_ENFORCE(output_var != nullptr, "output link [%s] is not in scope.",
                    outlinks[i].external);
-    Tensor* output = output_var->GetMutable<Tensor>();
+    LoDTensor* output = output_var->GetMutable<LoDTensor>();
 
     if (infer_shape_mode) {
       auto step_scope_var = step_scopes[0]->FindVar(outlinks[i].internal);
       PADDLE_ENFORCE(step_scope_var != nullptr, "%s not in scope",
                      outlinks[i].internal);
-      f::DDim step_dims = step_scope_var->template GetMutable<Tensor>()->dims();
+      f::DDim step_dims =
+          step_scope_var->template GetMutable<LoDTensor>()->dims();
       std::vector<int64_t> dims_vec = vectorize(step_dims);
       dims_vec.insert(dims_vec.begin(), seq_len);
       output->Resize(f::make_ddim(dims_vec));
     } else {
       output->mutable_data<float>(platform::CPUPlace());
       for (size_t j = 0; j < seq_len; j++) {
-        Tensor* step_output =
-            step_scopes[j]->FindVar(outlinks[i].internal)->GetMutable<Tensor>();
+        LoDTensor* step_output = step_scopes[j]
+                                     ->FindVar(outlinks[i].internal)
+                                     ->GetMutable<LoDTensor>();
         // TODO(luotao02) data type and platform::DeviceContext() should set
         // correctly
         (output->Slice<float>(j, j + 1))
@@ -94,8 +99,8 @@ void LinkMemories(const std::vector<Scope*>& scopes,
   auto scope = scopes[step_id];
   auto linked_scope = scopes[step_id + offset];
   for (auto& attr : memories) {
-    auto mem = scope->FindVar(attr.pre_var)->GetMutable<Tensor>();
-    auto linked_mem = linked_scope->FindVar(attr.var)->GetMutable<Tensor>();
+    auto mem = scope->FindVar(attr.pre_var)->GetMutable<LoDTensor>();
+    auto linked_mem = linked_scope->FindVar(attr.var)->GetMutable<LoDTensor>();
     if (infer_shape_mode) {
       mem->Resize(linked_mem->dims());
     } else {
diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc
index fa8f0ff1a858143af427b51025279c726f1628e0..c6101685a3205a7a7459347ea5b0cc8487656550 100644
--- a/paddle/operators/rowwise_add_op.cc
+++ b/paddle/operators/rowwise_add_op.cc
@@ -37,7 +37,7 @@ class RowwiseAddOp : public framework::OperatorWithKernel {
         framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims,
         "The width of two operands must be same");
     PADDLE_ENFORCE_EQ(ctx.OutputSize("Out"), 1, "The output size must be 1");
-    ctx.Output<Tensor>("Out")->Resize(x_dims);
+    ctx.Output<framework::LoDTensor>("Out")->Resize(x_dims);
   }
 };
 
@@ -76,8 +76,8 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(
         framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims,
         "The width of two operands must be same");
-    auto *dx = ctx.Output<Tensor>(framework::GradVarName("X"));
-    auto *db = ctx.Output<Tensor>(framework::GradVarName("b"));
+    auto *dx = ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    auto *db = ctx.Output<framework::LoDTensor>(framework::GradVarName("b"));
     if (dx) dx->Resize(x_dims);
     if (db) db->Resize(b_dims);
   }
diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc
index ea991f683d841b3dc4624a0d8aa3c88367fd3c6d..35e6b70ba94a645da2b99093b2354acfb0ef2771 100644
--- a/paddle/operators/scale_op.cc
+++ b/paddle/operators/scale_op.cc
@@ -28,7 +28,7 @@ class ScaleOp : public framework::OperatorWithKernel {
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
     auto *in = ctx.Input<framework::Tensor>("X");
-    auto *out = ctx.Output<framework::Tensor>("Out");
+    auto *out = ctx.Output<framework::LoDTensor>("Out");
     out->Resize(in->dims());
   }
 };
diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc
index f901edefa22dc9a252e87116df756d04767a7162..0f7510983e2e34485110719c92d26cbc78cd850c 100644
--- a/paddle/operators/scatter_op.cc
+++ b/paddle/operators/scatter_op.cc
@@ -35,7 +35,8 @@ class ScatterOp : public framework::OperatorWithKernel {
     framework::DDim data_dim(ctx.Input<Tensor>("Updates")->dims());
     for (int i = 1; i < data_dim.size(); ++i)
       PADDLE_ENFORCE_EQ(data_dim[i], ctx.Input<Tensor>("Updates")->dims()[i]);
-    ctx.Output<Tensor>("Out")->Resize(ctx.Input<Tensor>("Ref")->dims());
+    ctx.Output<framework::LoDTensor>("Out")->Resize(
+        ctx.Input<Tensor>("Ref")->dims());
   }
 };
 
@@ -45,9 +46,11 @@ class ScatterGradOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    auto *dUpdates = ctx.Output<Tensor>(framework::GradVarName("Updates"));
+    auto *dUpdates =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Updates"));
     auto *Updates = ctx.Input<Tensor>("Updates");
-    auto *dRef = ctx.Output<Tensor>(framework::GradVarName("Ref"));
+    auto *dRef =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Ref"));
     auto *Ref = ctx.Input<Tensor>("Ref");
 
     dRef->Resize(Ref->dims());
diff --git a/paddle/operators/sequence_avg_pool_op.cc b/paddle/operators/sequence_avg_pool_op.cc
new file mode 100644
index 0000000000000000000000000000000000000000..c15a5833deba2e198f6cb724bda7e3306c56e461
--- /dev/null
+++ b/paddle/operators/sequence_avg_pool_op.cc
@@ -0,0 +1,90 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/sequence_avg_pool_op.h"
+
+namespace paddle {
+namespace operators {
+
+class SequenceAvgPoolOp : public framework::OperatorWithKernel {
+ public:
+  using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+  void InferShape(const framework::InferShapeContext& ctx) const override {
+    PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"),
+                            "Input of SequenceAvgPoolOp"
+                            "must be initialized.");
+    auto* x = ctx.Input<framework::LoDTensor>("X");
+    auto dims = x->dims();
+    auto lod = x->lod();
+    PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now.");
+    PADDLE_ENFORCE_GE(
+        dims[0],
+        /*batch size = */ static_cast<int64_t>(lod[0].size() - 1),
+        "The first dimension of Input(X) must be large than batch size.");
+    dims[0] = lod[0].size() - 1;
+    ctx.Output<framework::LoDTensor>("Out")->Resize({dims});
+  }
+};
+
+class SequenceAvgPoolOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  SequenceAvgPoolOpMaker(framework::OpProto* proto,
+                         framework::OpAttrChecker* op_checker)
+      : OpProtoAndCheckerMaker(proto, op_checker) {
+    AddInput("X", "Input of SequenceAvgPoolOp.");
+    AddOutput("Out", "The output of SequenceAvgPoolOp.");
+    AddComment(R"DOC(
+    SequenceAvgPoolOp averages features of all time-steps of each instance.
+    More detailed comments will be added later.
+    )DOC");
+  }
+};
+
+class SequenceAvgPoolGradOp : public framework::OperatorWithKernel {
+ public:
+  using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+  void InferShape(const framework::InferShapeContext& ctx) const override {
+    PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")),
+                            "Gradient of Out should not be null");
+    auto og_dims =
+        ctx.Input<framework::LoDTensor>(framework::GradVarName("Out"))->dims();
+    auto x_dims = ctx.Input<framework::LoDTensor>("X")->dims();
+    PADDLE_ENFORCE_EQ(og_dims.size(), x_dims.size(),
+                      "The rank of output grad must equal to Input(X).");
+    for (int64_t i = 1; i < og_dims.size(); ++i) {
+      PADDLE_ENFORCE_EQ(og_dims[i], x_dims[i], "The dimension mismatch.");
+    }
+    auto* x_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    x_grad->Resize(x_dims);
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP(sequence_avg_pool, ops::SequenceAvgPoolOp,
+            ops::SequenceAvgPoolOpMaker, sequence_avg_pool_grad,
+            ops::SequenceAvgPoolGradOp);
+REGISTER_OP_CPU_KERNEL(
+    sequence_avg_pool,
+    ops::SequenceAvgPoolKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+    sequence_avg_pool_grad,
+    ops::SequenceAvgPoolGradKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/sequence_avg_pool_op.cu b/paddle/operators/sequence_avg_pool_op.cu
new file mode 100644
index 0000000000000000000000000000000000000000..bc9d1611fccd17c99b914b6ef59995288a9ebbd6
--- /dev/null
+++ b/paddle/operators/sequence_avg_pool_op.cu
@@ -0,0 +1,25 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#define EIGEN_USE_GPU
+
+#include "paddle/operators/sequence_avg_pool_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(
+    sequence_avg_pool,
+    ops::SequenceAvgPoolKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+    sequence_avg_pool_grad,
+    ops::SequenceAvgPoolGradKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/sequence_avg_pool_op.h b/paddle/operators/sequence_avg_pool_op.h
new file mode 100644
index 0000000000000000000000000000000000000000..6e343b87e2938399409498407ac46b2416dc2231
--- /dev/null
+++ b/paddle/operators/sequence_avg_pool_op.h
@@ -0,0 +1,81 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+using Tensor = framework::Tensor;
+using LoDTensor = framework::LoDTensor;
+template <typename T, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+using EigenMatrix = framework::EigenMatrix<T, MajorType, IndexType>;
+
+template <typename Place, typename T>
+class SequenceAvgPoolKernel : public framework::OpKernel {
+ public:
+  void Compute(const framework::ExecutionContext& context) const override {
+    auto* in = context.Input<LoDTensor>("X");
+    auto* out = context.Output<LoDTensor>("Out");
+
+    auto dims = in->dims();
+    auto lod = in->lod();
+    int64_t w = in->numel() / dims[0];
+
+    out->mutable_data<T>(context.GetPlace());
+    auto place = context.GetEigenDevice<Place>();
+    for (int i = 0; i < static_cast<int>(lod[0].size()) - 1; ++i) {
+      Tensor in_t = in->Slice<T>(static_cast<int>(lod[0][i]),
+                                 static_cast<int>(lod[0][i + 1]));
+      Tensor out_t = out->Slice<T>(i, i + 1);
+      int64_t h = static_cast<int64_t>(lod[0][i + 1] - lod[0][i]);
+      auto in_e = EigenMatrix<T>::From(in_t, {h, w});
+      auto out_e = EigenMatrix<T>::From(out_t, {h, w});
+      out_e.device(place) = in_e.mean(Eigen::array<int, 1>({{0}}));
+    }
+  }
+};
+
+template <typename Place, typename T>
+class SequenceAvgPoolGradKernel : public framework::OpKernel {
+ public:
+  void Compute(const framework::ExecutionContext& context) const override {
+    auto* in = context.Output<LoDTensor>("X");
+    auto* in_g = context.Output<LoDTensor>(framework::GradVarName("X"));
+    auto* out_g = context.Input<LoDTensor>(framework::GradVarName("Out"));
+
+    auto dims = in->dims();
+    auto lod = in->lod();
+    int64_t w = in->numel() / dims[0];
+
+    in_g->mutable_data<T>(context.GetPlace());
+    auto place = context.GetEigenDevice<Place>();
+    for (int i = 0; i < static_cast<int>(lod[0].size()) - 1; ++i) {
+      auto in_g_t = in_g->Slice<T>(static_cast<int>(lod[0][i]),
+                                   static_cast<int>(lod[0][i + 1]));
+      auto out_g_t = out_g->Slice<T>(i, i + 1);
+      int64_t h = static_cast<int64_t>(lod[0][i + 1] - lod[0][i]);
+      auto in_g_e = EigenMatrix<T>::From(in_g_t, {h, w});
+      auto out_g_e = EigenMatrix<T>::From(out_g_t, {1, w});
+      Eigen::DSizes<int, 2> bcast(h, w);
+      in_g_e.device(place) = (out_g_e / static_cast<T>(h)).broadcast(bcast);
+    }
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle
diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc
index ad267e7f087943ff3b8326a7baf2ce3955fa51c2..7997bf690710b3675f4014790db8cc7fc06946d3 100644
--- a/paddle/operators/sgd_op.cc
+++ b/paddle/operators/sgd_op.cc
@@ -23,10 +23,11 @@ class SGDOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    PADDLE_ENFORCE(
-        ctx.Input<Tensor>("param")->dims() == ctx.Input<Tensor>("grad")->dims(),
-        "Two input of SGD Op's dimension must be same.");
-    ctx.Output<Tensor>("param_out")->Resize(ctx.Input<Tensor>("param")->dims());
+    PADDLE_ENFORCE_EQ(ctx.Input<Tensor>("param")->dims(),
+                      ctx.Input<Tensor>("grad")->dims(),
+                      "Two input of SGD Op's dimension must be same.");
+    ctx.Output<framework::LoDTensor>("param_out")
+        ->Resize(ctx.Input<Tensor>("param")->dims());
   }
 };
 
diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc
index 761c6de8d4d2150b30b97b58da95da3d5f33db63..de6a1ba77382306923ee238e5cf309c791f9f216 100644
--- a/paddle/operators/sigmoid_op.cc
+++ b/paddle/operators/sigmoid_op.cc
@@ -23,7 +23,8 @@ class SigmoidOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    ctx.Output<Tensor>("Y")->Resize(ctx.Input<Tensor>("X")->dims());
+    ctx.Output<framework::LoDTensor>("Y")->Resize(
+        ctx.Input<Tensor>("X")->dims());
   }
 };
 
@@ -44,7 +45,7 @@ class SigmoidOpGrad : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    ctx.Output<Tensor>(framework::GradVarName("X"))
+    ctx.Output<framework::LoDTensor>(framework::GradVarName("X"))
         ->Resize(ctx.Input<Tensor>("Y")->dims());
   }
 };
diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc
index 7166b2f60be8a6088ab3a81686f7bed1b7181d97..239d3d141e1076a0a6a943f340311b17aa6f542a 100644
--- a/paddle/operators/softmax_op.cc
+++ b/paddle/operators/softmax_op.cc
@@ -25,7 +25,8 @@ class SoftmaxOp : public framework::OperatorWithKernel {
   void InferShape(const framework::InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.Input<Tensor>("X")->dims().size() == 2UL,
                    "The input of softmax op must be a matrix.");
-    ctx.Output<Tensor>("Y")->Resize(ctx.Input<Tensor>("X")->dims());
+    ctx.Output<framework::LoDTensor>("Y")->Resize(
+        ctx.Input<Tensor>("X")->dims());
   }
 };
 
@@ -71,7 +72,7 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
                       ctx.Input<Tensor>(framework::GradVarName("Y"))->dims(),
                       "Input(Y) and its gradients should have a same shape.");
 
-    ctx.Output<Tensor>(framework::GradVarName("X"))
+    ctx.Output<framework::LoDTensor>(framework::GradVarName("X"))
         ->Resize(ctx.Input<Tensor>("X")->dims());
   }
 };
diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc
index 9f51d3efa8ecba894a1023b9de2df451ca85916c..ebe5bd352e99e298fb86355730feed77b236d2bd 100644
--- a/paddle/operators/squared_l2_distance_op.cc
+++ b/paddle/operators/squared_l2_distance_op.cc
@@ -48,9 +48,9 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel {
                    "First dimension of target must be equal to input "
                    "or to 1.");
 
-    ctx.Output<Tensor>("sub_result")
+    ctx.Output<framework::LoDTensor>("sub_result")
         ->Resize({x_dims[0], x->numel() / x_dims[0]});
-    ctx.Output<Tensor>("Out")->Resize({x_dims[0], 1});
+    ctx.Output<framework::LoDTensor>("Out")->Resize({x_dims[0], 1});
   }
 };
 
@@ -94,8 +94,10 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(out_dims[1], 1,
                       "Second dimension of output gradient "
                       "must be 1.");
-    auto* x_grad = ctx.Output<Tensor>(framework::GradVarName("X"));
-    auto* y_grad = ctx.Output<Tensor>(framework::GradVarName("Y"));
+    auto* x_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("X"));
+    auto* y_grad =
+        ctx.Output<framework::LoDTensor>(framework::GradVarName("Y"));
     if (x_grad) x_grad->Resize(x_dims);
     if (y_grad) y_grad->Resize(y_dims);
   }
diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc
index 5805826ee8a555ca6dfc1ca81feaadffea9e1012..7170e7256c206d338ef1f6f94d5d1889ca92a3de 100644
--- a/paddle/operators/sum_op.cc
+++ b/paddle/operators/sum_op.cc
@@ -23,7 +23,7 @@ class SumOp : public framework::OperatorWithKernel {
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
     auto ins = ctx.MultiInput<framework::Tensor>("X");
-    auto *out = ctx.Output<framework::Tensor>("Out");
+    auto *out = ctx.Output<framework::LoDTensor>("Out");
     int N = ins.size();
 
     auto in_dim = ins[0]->dims();
@@ -55,7 +55,8 @@ class SumGradOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
-    auto outputs = ctx.MultiOutput<Tensor>(framework::GradVarName("X"));
+    auto outputs =
+        ctx.MultiOutput<framework::LoDTensor>(framework::GradVarName("X"));
     auto dims = ctx.Input<Tensor>(framework::GradVarName("Out"))->dims();
     for (auto output : outputs) {
       output->Resize(dims);
diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc
index 38d2f0a09aec751734864947a2f3cfa20107e22f..ff0e77a344ede7709a805d7dca4397eb49fa300c 100644
--- a/paddle/operators/top_k_op.cc
+++ b/paddle/operators/top_k_op.cc
@@ -35,8 +35,8 @@ class TopkOp : public framework::OperatorWithKernel {
 
     framework::DDim dims = input->dims();
     dims[dims.size() - 1] = k;
-    ctx.Output<Tensor>("Out")->Resize(dims);
-    ctx.Output<Tensor>("Indices")->Resize(dims);
+    ctx.Output<framework::LoDTensor>("Out")->Resize(dims);
+    ctx.Output<framework::LoDTensor>("Indices")->Resize(dims);
   }
 };
 
diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc
index b8fbc9b52aecdb5c8d985b5de9bcd7cb85835b60..ed7973693619e7765643dda824100afd82616470 100644
--- a/paddle/operators/uniform_random_op.cc
+++ b/paddle/operators/uniform_random_op.cc
@@ -50,7 +50,7 @@ class UniformRandomOp : public framework::OperatorWithKernel {
   void InferShape(const framework::InferShapeContext& ctx) const override {
     PADDLE_ENFORCE(Attr<float>("min") < Attr<float>("max"),
                    "uniform_random's min must less then max");
-    auto* tensor = ctx.Output<framework::Tensor>("Out");
+    auto* tensor = ctx.Output<framework::LoDTensor>("Out");
     auto dims = Attr<std::vector<int>>("dims");
     std::vector<int64_t> temp;
     temp.reserve(dims.size());
diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc
index 599a25f04c8b391648219b1944ba4378690f2cae..a7a38339fb2c8689778b0a86d3713f67e1447a80 100644
--- a/paddle/pybind/pybind.cc
+++ b/paddle/pybind/pybind.cc
@@ -97,27 +97,21 @@ PYBIND11_PLUGIN(core) {
         return self.data<float>()[offset];
       });
 
-  py::class_<LoDTensor>(m, "LoDTensor", R"DOC(LoD(Leval of Ddetails) Tensor.
-
-The tensor and LoD info should be created before creating the LoDTensor, then
-call the set_tensor and set_lod functions to set them.
-
-)DOC")
-      .def("__init__",
-           [](LoDTensor &instance,
-              const std::vector<std::vector<size_t>> &lod,
-              Tensor *t) {
+  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
+      .def_buffer(
+          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
+      .def(
+          "__init__",
+          [](LoDTensor &instance, const std::vector<std::vector<size_t>> &lod) {
 #ifdef PADDLE_ONLY_CPU
-             new (&instance) LoDTensor(lod, t);
+            new (&instance) LoDTensor(lod);
 #else
              paddle::framework::LoD new_lod;
              new_lod.reserve(lod.size());
              std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
-             new (&instance) LoDTensor(new_lod, t);
+             new (&instance) LoDTensor(new_lod);
 #endif
-           })
-      .def("set_tensor",
-           [](LoDTensor &self, Tensor *tensor) { self.set_tensor(tensor); })
+          })
       .def("set_lod",
            [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
 #ifdef PADDLE_ONLY_CPU
@@ -129,9 +123,6 @@ call the set_tensor and set_lod functions to set them.
              self.set_lod(new_lod);
 #endif
            })
-      .def("tensor",
-           [](LoDTensor &self) -> Tensor & { return self.tensor(); },
-           py::return_value_policy::reference)
       .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
 #ifdef PADDLE_ONLY_CPU
         return self.lod();
@@ -160,9 +151,6 @@ All parameter, weight, gradient are variables in Paddle.
            [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
       .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
       .def("get_tensor",
-           [](Variable &self) -> Tensor * { return self.GetMutable<Tensor>(); },
-           py::return_value_policy::reference)
-      .def("get_lod_tensor",
            [](Variable &self) -> LoDTensor * {
              return self.GetMutable<LoDTensor>();
            },
diff --git a/python/paddle/v2/framework/tests/test_tensor.py b/python/paddle/v2/framework/tests/test_tensor.py
index f26ed4964c521be1cd839b39d7244f96c653cb1a..8cd93b35d7d1cb7d3b4a19e0e402ef576f1c0982 100644
--- a/python/paddle/v2/framework/tests/test_tensor.py
+++ b/python/paddle/v2/framework/tests/test_tensor.py
@@ -44,79 +44,66 @@ class TestTensor(unittest.TestCase):
         self.assertAlmostEqual(2.0, tensor_array_2[19, 11])
 
     def test_int_lod_tensor(self):
-        places = [core.CPUPlace(), core.GPUPlace(0)]
-        for place in places:
-            scope = core.Scope()
-            var = scope.new_var("test_tensor")
-            var_lod = scope.new_var("test_lod_tensor")
-
-            tensor = var.get_tensor()
-            lod_tensor = var_lod.get_lod_tensor()
-
-            tensor.set_dims([4, 4, 6])
-            tensor.alloc_int(place)
-            array = numpy.array(tensor)
-            array[0, 0, 0] = 3
-            array[3, 3, 5] = 10
-            tensor.set(array, place)
+        place = core.CPUPlace()
+        scope = core.Scope()
+        var_lod = scope.new_var("test_lod_tensor")
+        lod_tensor = var_lod.get_tensor()
 
-            lod_tensor.set_tensor(tensor)
-            lod_tensor.set_lod([[0, 2, 4]])
+        lod_tensor.set_dims([4, 4, 6])
+        lod_tensor.alloc_int(place)
+        array = numpy.array(lod_tensor)
+        array[0, 0, 0] = 3
+        array[3, 3, 5] = 10
+        lod_tensor.set(array, place)
+        lod_tensor.set_lod([[0, 2, 4]])
 
-            lod_v = numpy.array(lod_tensor.tensor())
-            self.assertTrue(numpy.alltrue(array == lod_v))
+        lod_v = numpy.array(lod_tensor)
+        self.assertTrue(numpy.alltrue(array == lod_v))
 
-            lod = lod_tensor.lod()
-            self.assertEqual(0, lod[0][0])
-            self.assertEqual(2, lod[0][1])
-            self.assertEqual(4, lod[0][2])
+        lod = lod_tensor.lod()
+        self.assertEqual(0, lod[0][0])
+        self.assertEqual(2, lod[0][1])
+        self.assertEqual(4, lod[0][2])
 
     def test_float_lod_tensor(self):
-        places = [core.CPUPlace(), core.GPUPlace(0)]
-        for place in places:
-            scope = core.Scope()
-            var = scope.new_var("test_tensor")
-            var_lod = scope.new_var("test_lod_tensor")
-
-            tensor = var.get_tensor()
-            lod_tensor = var_lod.get_lod_tensor()
-
-            tensor.set_dims([5, 2, 3, 4])
-            tensor.alloc_float(place)
+        place = core.CPUPlace()
+        scope = core.Scope()
+        var_lod = scope.new_var("test_lod_tensor")
 
-            tensor_array = numpy.array(tensor)
-            self.assertEqual((5, 2, 3, 4), tensor_array.shape)
-            tensor_array[0, 0, 0, 0] = 1.0
-            tensor_array[0, 0, 0, 1] = 2.0
-            tensor.set(tensor_array, place)
+        lod_tensor = var_lod.get_tensor()
+        lod_tensor.set_dims([5, 2, 3, 4])
+        lod_tensor.alloc_float(place)
 
-            lod_tensor.set_tensor(tensor)
+        tensor_array = numpy.array(lod_tensor)
+        self.assertEqual((5, 2, 3, 4), tensor_array.shape)
+        tensor_array[0, 0, 0, 0] = 1.0
+        tensor_array[0, 0, 0, 1] = 2.0
+        lod_tensor.set(tensor_array, place)
 
-            lod_v = numpy.array(lod_tensor.tensor())
-            self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0])
-            self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1])
-            self.assertEqual(len(lod_tensor.lod()), 0)
+        lod_v = numpy.array(lod_tensor)
+        self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0])
+        self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1])
+        self.assertEqual(len(lod_tensor.lod()), 0)
 
-            lod_py = [[0, 2, 5], [0, 2, 4, 5]]
-            lod_tensor.set_lod(lod_py)
-            lod = lod_tensor.lod()
-            self.assertListEqual(lod_py, lod)
+        lod_py = [[0, 2, 5], [0, 2, 4, 5]]
+        lod_tensor.set_lod(lod_py)
+        lod = lod_tensor.lod()
+        self.assertListEqual(lod_py, lod)
 
     def test_lod_tensor_init(self):
         scope = core.Scope()
-        var = scope.new_var("test_tensor")
         place = core.CPUPlace()
-        tensor = var.get_tensor()
-        tensor.set_dims([5, 2, 3, 4])
-        tensor.alloc_float(place)
-        tensor_array = numpy.array(tensor)
+        lod_py = [[0, 2, 5], [0, 2, 4, 5]]
+        lod_tensor = core.LoDTensor(lod_py)
+
+        lod_tensor.set_dims([5, 2, 3, 4])
+        lod_tensor.alloc_float(place)
+        tensor_array = numpy.array(lod_tensor)
         tensor_array[0, 0, 0, 0] = 1.0
         tensor_array[0, 0, 0, 1] = 2.0
-        tensor.set(tensor_array, place)
-        lod_py = [[0, 2, 5], [0, 2, 4, 5]]
+        lod_tensor.set(tensor_array, place)
 
-        lod_tensor = core.LoDTensor(lod_py, tensor)
-        lod_v = numpy.array(lod_tensor.tensor())
+        lod_v = numpy.array(lod_tensor)
         self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0])
         self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1])
         self.assertListEqual(lod_py, lod_tensor.lod())