Refactor dygraph to eager -- TensorWrapper, EagerUtils, GlobalUtils (#37466)

* Add EagerTensor and tests

* remove useless enforce

* remove comment in cmake

* support autograd meta

* support grad node info test

* support grad_node_info

* add more edge test

* remove Python.h

* add tensor wrapper with tests

* support compute require grad and stop gradient

* support sync methods and global utils

* support pure cpu test

* refine error msg

* refine error msg

* refine error info

* fix npu error

paddle/fluid/eager/CMakeLists.txt

 add_subdirectory(tests)
+add_subdirectory(api)
 cc_library(grad_node_info SRCS grad_node_info.cc DEPS pten pten_api)
 cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api)
+cc_library(utils SRCS utils.cc DEPS pten pten_api autograd_meta eager_api)

paddle/fluid/eager/api/CMakeLists.txt

+add_subdirectory(utils)
+
+cc_library(eager_api SRCS all.cc DEPS global_utils)

paddle/fluid/eager/api/all.cc

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "paddle/fluid/eager/api/all.h"
+
+namespace egr {}  // namespace egr

paddle/fluid/eager/api/all.h

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#pragma once
+
+#include "paddle/fluid/eager/api/utils/global_utils.h"

paddle/fluid/eager/api/utils/CMakeLists.txt

+cc_library(global_utils SRCS global_utils.cc DEPS enforce)

paddle/fluid/eager/api/utils/global_utils.cc

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+
+namespace egr {
+
+Controller* Controller::controller_ = new Controller();
+
+}  // namespace egr

paddle/fluid/eager/api/utils/global_utils.h

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace egr {
+
+class UniqueNameGenerator {
+ public:
+  explicit UniqueNameGenerator(std::string prefix = "") : prefix_(prefix) {}
+  std::string Generate(std::string key = "eager_tmp") {
+    return prefix_ + key + "_" + std::to_string(id_++);
+  }
+
+ private:
+  std::atomic<int> id_{0};
+  std::string prefix_;
+};
+
+// Global
+class Controller {
+ public:
+  static Controller& Instance() { return *controller_; }
+  const paddle::platform::Place& GetExpectedPlace() const {
+    return *expected_place_.get();
+  }
+  void SetExpectedPlace(const paddle::platform::Place& place) {
+    expected_place_ = std::make_shared<paddle::platform::Place>(place);
+  }
+  void SetAMPLevel(int level) { amp_level_ = level; }
+  int GetAMPLevel() const { return amp_level_; }
+  bool HasGrad() const { return has_grad_; }
+  std::string GenerateUniqueName(std::string key = "eager_tmp") {
+    return generator_->Generate(key);
+  }
+
+ private:
+  Controller() = default;
+  static Controller* controller_;
+  std::shared_ptr<paddle::platform::Place> expected_place_ = nullptr;
+  int amp_level_ = 0;
+  bool has_grad_ = true;
+  std::unique_ptr<UniqueNameGenerator> generator_{new UniqueNameGenerator()};
+  DISABLE_COPY_AND_ASSIGN(Controller);
+};
+
+}  // namespace egr

paddle/fluid/eager/eager_tensor.h

@@ -14,7 +14,6 @@
 
 #pragma once
 // framework deps
-#include "paddle/fluid/framework/data_layout_transform.h"
 #include "paddle/fluid/framework/pten_utils.h"
 #include "paddle/fluid/framework/tensor.h"
 #include "paddle/fluid/framework/variable.h"

paddle/fluid/eager/tensor_wrapper.h

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * We now still need TensorWrapper and it is designed to Copy
+ * tensor in autograd mode.
+ *
+ * Since in autograd usage, we need to pass autograd_meta to
+ * backward computation however in tensor interface add to much
+ * autograd_related method is not a good choice.
+ *
+ * In TensorWrapper we will keep autograd info to backward, only
+ * for input var, but for output var it will only copy autograd
+ * with no grad **/
+
+#pragma once
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/utils.h"
+
+namespace egr {
+class TensorWrapper {
+ public:
+  TensorWrapper() = default;
+  explicit TensorWrapper(const egr::EagerTensor& tensor,
+                         bool full_reserved = false) {
+    /**
+     * Normally, we should fully reserved all non-output or non-leaf fwd tensor
+     * here. And for fwd output tensor, we should not reserve its autogradmeta,
+     * to avoid recursive depends on GradNodeBase
+     * **/
+    full_reserved_ = full_reserved;
+    if (full_reserved_) {
+      VLOG(6) << "Fully reserved tensor: " << tensor.name();
+      intermidiate_tensor_ = tensor;
+      return;
+    }
+
+    // shallow copy tensor_impl here
+    intermidiate_tensor_.set_impl(tensor.impl());
+    intermidiate_tensor_.ResetVar(tensor.Var());
+    intermidiate_tensor_.set_name(tensor.name() + "@Saved");
+    PADDLE_ENFORCE_NOT_NULL(
+        EagerUtils::unsafe_autograd_meta(tensor),
+        paddle::platform::errors::Fatal(
+            "Full reserved Tensor should not have null autograd meta, since "
+            "tensor_wrapper is used to build backward info. There is no way "
+            "for us to build it with null autograd_meta."));
+    // copy output_rank
+    out_rank_info_ = EagerUtils::OutRankInfo(tensor);
+  }
+
+  egr::EagerTensor recover(const std::shared_ptr<GradNodeBase>& grad_node) {
+    VLOG(6) << "Recover tensor for wrapper";
+    if ((!intermidiate_tensor_.defined()) &&
+        (!intermidiate_tensor_.Var().IsInitialized())) {
+      VLOG(6) << "Return NULL tensor Here. ";
+      return egr::EagerTensor();
+    }
+
+    // if it's full_reserved just return the full copy of tensor
+    if (full_reserved_) {
+      return intermidiate_tensor_;
+    } else {
+      std::shared_ptr<GradNodeBase> new_grad_node = grad_node;
+      auto p_ab_autograd_meta =
+          std::make_shared<AutogradMeta>(Edge(new_grad_node, out_rank_info_));
+      intermidiate_tensor_.set_autograd_meta(
+          std::static_pointer_cast<paddle::experimental::AbstractAutogradMeta>(
+              p_ab_autograd_meta));
+      return intermidiate_tensor_;
+    }
+  }
+
+ private:
+  bool full_reserved_ = false;
+  std::pair<size_t, size_t> out_rank_info_;
+  egr::EagerTensor intermidiate_tensor_;
+};
+}  // namespace egr

paddle/fluid/eager/tests/CMakeLists.txt

 set(eager_deps pten pten_api)
 add_subdirectory(data_structure_tests)
+add_subdirectory(task_tests)

paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt

 cc_test(test_egr_ds_eager_tensor SRCS eager_tensor_test.cc DEPS ${eager_deps} )
 cc_test(test_egr_ds_auotgrad_meta SRCS autograd_meta_test.cc DEPS ${eager_deps} grad_node_info)
 cc_test(test_egr_ds_grad_node_info SRCS grad_node_info_test.cc DEPS ${eager_deps} grad_node_info)
+cc_test(test_egr_ds_tensor_wrapper SRCS tensor_wrapper_test.cc DEPS ${eager_deps} grad_node_info utils)

paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h

@@ -19,6 +19,9 @@
 #include "paddle/fluid/eager/eager_tensor.h"
 #include "paddle/fluid/eager/grad_node_info.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
+namespace egr {
+class TensorWrapper;
+}
 
 namespace eager_test {
 class GradTestNode : public egr::GradNodeBase {

paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "paddle/fluid/eager/tensor_wrapper.h"
+#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
+#include "paddle/fluid/eager/utils.h"
+
+TEST(TensorWrapper, Basic) {
+  VLOG(6) << "Test Full reserved";
+  egr::EagerTensor et1;
+  pten::DenseTensorMeta meta = pten::DenseTensorMeta(
+      pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 2}));
+  std::shared_ptr<pten::DenseTensor> dt = std::make_shared<pten::DenseTensor>(
+      std::make_shared<paddle::experimental::DefaultAllocator>(
+          paddle::platform::CPUPlace()),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>();
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  et1.set_impl(dt);
+  // Create grad node;
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 2, /* out_num */ 2);
+  egr::Edge edge0(grad_test_node0, 1, 2);
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>(edge0);
+  et1.set_autograd_meta(auto_grad0);
+  et1.set_name("et1");
+  auto tw0 = egr::TensorWrapper(et1, true);
+  auto recover_et1 = tw0.recover(std::make_shared<eager_test::GradTestNode>());
+  CHECK_EQ(recover_et1.name(), std::string("et1"));
+  CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et1).first,
+           egr::EagerUtils::OutRankInfo(et1).first);
+  CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et1).second,
+           egr::EagerUtils::OutRankInfo(et1).second);
+  VLOG(6) << "Test reconstruct";
+  egr::EagerTensor et2;
+  pten::DenseTensorMeta meta2 = pten::DenseTensorMeta(
+      pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 2}));
+  std::shared_ptr<pten::DenseTensor> dt2 = std::make_shared<pten::DenseTensor>(
+      std::make_shared<paddle::experimental::DefaultAllocator>(
+          paddle::platform::CPUPlace()),
+      meta2);
+  auto* dt_ptr2 = dt->mutable_data<float>();
+  dt_ptr2[0] = 6.0f;
+  dt_ptr2[1] = 11.0f;
+  et2.set_impl(dt2);
+  et2.set_name("et2");
+  auto grad_test_node1 =
+      std::make_shared<eager_test::GradTestNode>(/* val */ 5.0, 2, 2);
+  egr::Edge edge1(grad_test_node1, 1, 2);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>(edge1);
+  et2.set_autograd_meta(auto_grad1);
+  auto tw1 = egr::TensorWrapper(et2, false);
+  auto recover_et2 = tw1.recover(grad_test_node1);
+  CHECK_EQ(recover_et2.name(), std::string("et2@Saved"));
+  CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et2).first,
+           egr::EagerUtils::OutRankInfo(et2).first);
+  CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et2).second,
+           egr::EagerUtils::OutRankInfo(et2).second);
+  // Test Raw recover
+  egr::EagerTensor et3;
+  auto tw2 = egr::TensorWrapper(et3, true);
+  CHECK(
+      tw2.recover(std::make_shared<eager_test::GradTestNode>()).initialized() ==
+      false);
+}

paddle/fluid/eager/tests/task_tests/CMakeLists.txt

+cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps} grad_node_info autograd_meta utils)

paddle/fluid/eager/tests/task_tests/eager_utils_test.cc

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Eager Dygraph
+
+#include "gtest/gtest.h"
+
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h"
+#include "paddle/fluid/eager/tests/test_utils.h"
+#include "paddle/fluid/eager/utils.h"
+
+namespace eager_test {
+template <typename T>
+egr::EagerTensor CreateTestCPUTensor(T val,
+                                     const paddle::framework::DDim& ddim) {
+  pten::DenseTensorMeta meta =
+      pten::DenseTensorMeta(pten::DataType::FLOAT32, ddim);
+  egr::EagerTensor tensor;
+  std::shared_ptr<pten::DenseTensor> dt = std::make_shared<pten::DenseTensor>(
+      std::make_shared<paddle::experimental::DefaultAllocator>(
+          paddle::platform::CPUPlace()),
+      meta);
+  auto* dt_ptr = dt->mutable_data<T>();
+  for (int64_t i = 0; i < dt->numel(); i++) {
+    dt_ptr[i] = val;
+  }
+  tensor.set_impl(dt);
+  return tensor;
+}
+}  // namespace eager_test
+TEST(EagerUtils, ComputeRequireGrad) {
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
+  CHECK_EQ(auto_grad0->NumericStopGradient(), -1);
+  VLOG(6) << "Single Test ComputeRequireGrad";
+  auto_grad0->SetStopGradient(true);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()) == false);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()) == false);
+  auto_grad0->SetStopGradient(false);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()) == false);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()) == true);
+
+  VLOG(6) << "Multi Test ComputeRequireGrad";
+  auto_grad0->SetStopGradient(false);
+  auto_grad1->SetStopGradient(true);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get(),
+                                            auto_grad1.get()) == true);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get(),
+                                            auto_grad1.get()) == false);
+  auto_grad0->SetStopGradient(true);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get(),
+                                            auto_grad1.get()) == false);
+  CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get(),
+                                            auto_grad1.get()) == false);
+}
+
+TEST(EagerUtils, PassStopGradient) {
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
+  CHECK_EQ(auto_grad0->NumericStopGradient(), -1);
+  VLOG(6) << "Test PassStopGradient";
+  egr::EagerUtils::PassStopGradient(false, auto_grad0.get());
+  CHECK(auto_grad0->StopGradient() == false);
+  egr::EagerUtils::PassStopGradient(true, auto_grad0.get(), auto_grad1.get(),
+                                    auto_grad2.get(), auto_grad3.get());
+  CHECK(auto_grad0->StopGradient() == true);
+  CHECK(auto_grad1->StopGradient() == true);
+  CHECK(auto_grad2->StopGradient() == true);
+  CHECK(auto_grad3->StopGradient() == true);
+}
+
+TEST(EagerUtils, SyncToVarsSingle) {
+  paddle::framework::DDim ddim = paddle::framework::make_ddim({2, 4, 4, 4});
+  auto tensor = eager_test::CreateTestCPUTensor(5.0f, ddim);
+  std::vector<std::shared_ptr<egr::EagerTensor>> var_bases =
+      egr::EagerUtils::SyncToVars(tensor);
+
+  paddle::framework::Variable* var = var_bases[0]->MutableVar();
+  const auto& framework_tensor = var->Get<paddle::framework::LoDTensor>();
+
+  const float* ptr = framework_tensor.data<float>();
+  VLOG(6) << "Check Value for SyncToVarsSingle";
+  CHECK_EQ(framework_tensor.numel(), tensor.numel());
+
+  for (int i = 0; i < framework_tensor.numel(); i++) {
+    CHECK_EQ(ptr[i], 5.0f);
+  }
+}
+
+TEST(EagerUtils, SyncToVarsMultiple) {
+  paddle::framework::DDim ddim = paddle::framework::make_ddim({2, 4, 4, 4});
+  std::vector<egr::EagerTensor> tensors = {
+      eager_test::CreateTestCPUTensor(1.0f, ddim),
+      eager_test::CreateTestCPUTensor(2.0f, ddim)};
+
+  std::vector<std::shared_ptr<egr::EagerTensor>> var_bases =
+      egr::EagerUtils::SyncToVars(tensors);
+
+  {
+    paddle::framework::Variable* var = var_bases[0]->MutableVar();
+    const auto& framework_tensor = var->Get<paddle::framework::LoDTensor>();
+
+    const float* ptr = framework_tensor.data<float>();
+    CHECK_EQ(framework_tensor.numel(), tensors[0].numel());
+
+    for (int i = 0; i < framework_tensor.numel(); i++) {
+      CHECK_EQ(ptr[i], 1.0);
+    }
+  }
+
+  {
+    paddle::framework::Variable* var = var_bases[1]->MutableVar();
+    const auto& framework_tensor = var->Get<paddle::framework::LoDTensor>();
+
+    const float* ptr = framework_tensor.data<float>();
+    VLOG(6) << "Check Value for SyncToVarsMultiple";
+    CHECK_EQ(framework_tensor.numel(), tensors[0].numel());
+
+    for (int i = 0; i < framework_tensor.numel(); i++) {
+      CHECK_EQ(ptr[i], 2.0);
+    }
+  }
+}
+
+TEST(EagerUtils, SyncToTensorSingle) {
+  std::shared_ptr<egr::EagerTensor> X(new egr::EagerTensor());
+  std::vector<float> src_data(128, 5.0);
+  std::vector<int64_t> dims = {2, 4, 4, 4};
+  paddle::platform::CPUPlace place;
+
+  auto* x_tensor = X->MutableVar()->GetMutable<paddle::framework::LoDTensor>();
+  x_tensor->Resize(paddle::framework::make_ddim(dims));
+  auto* mutable_x = x_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place, mutable_x, place, src_data.data(),
+                       sizeof(float) * src_data.size());
+  auto X_ = egr::EagerUtils::SyncToTensors(*(X.get()));
+  egr::EagerTensor tensor = egr::EagerUtils::GetOutput(X_[0]);
+  VLOG(6) << "Check Value for SyncToTensorSingle";
+  CHECK(eager_test::CompareTensorWithValue<float>(tensor, 5.0));
+}
+
+TEST(EagerUtils, SyncToTensorMultiple) {
+  eager_test::InitEnv(paddle::platform::CPUPlace());
+  std::vector<int64_t> dims = {2, 4, 4, 4};
+  paddle::platform::CPUPlace place;
+
+  std::vector<egr::EagerTensor> egr_tensors;
+  {
+    auto egr_tensor = egr::EagerTensor();
+    std::vector<float> src_data(128, 1.0);
+    auto* x_tensor =
+        egr_tensor.MutableVar()->GetMutable<paddle::framework::LoDTensor>();
+    x_tensor->Resize(paddle::framework::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place, mutable_x, place, src_data.data(),
+                         sizeof(float) * src_data.size());
+    egr_tensors.emplace_back(egr_tensor);
+  }
+  {
+    auto egr_tensor = egr::EagerTensor();
+    std::vector<float> src_data(128, 2.0);
+    auto* x_tensor =
+        egr_tensor.MutableVar()->GetMutable<paddle::framework::LoDTensor>();
+    x_tensor->Resize(paddle::framework::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place, mutable_x, place, src_data.data(),
+                         sizeof(float) * src_data.size());
+    egr_tensors.emplace_back(std::move(egr_tensor));
+  }
+  std::vector<egr::EagerTensor> tensors =
+      egr::EagerUtils::GetOutputs(egr::EagerUtils::SyncToTensors(egr_tensors));
+
+  VLOG(6) << "Check Value for SyncToTensorMultiple";
+  CHECK(eager_test::CompareTensorWithValue<float>(tensors[0], 1.0) == true);
+  CHECK(eager_test::CompareTensorWithValue<float>(tensors[1], 2.0) == true);
+}
+
+TEST(EagerUtils, ConstructDuplicableOutput) {
+  VLOG(6) << "Check ConstructDuplicableOutput";
+  std::vector<std::shared_ptr<egr::EagerTensor>> outs =
+      egr::EagerUtils::ConstructDuplicableOutput(2);
+  CHECK_EQ(outs.size(), size_t(2));
+  CHECK(outs[0]->defined() == false);
+  CHECK(outs[0]->initialized() == false);
+}

paddle/fluid/eager/tests/test_utils.h

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/utils.h"
+
+#include "paddle/pten/api/all.h"
+#include "paddle/pten/core/dense_tensor.h"
+#include "paddle/pten/core/tensor_meta.h"
+
+#include "paddle/fluid/memory/memcpy.h"
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/init.h"
+
+namespace eager_test {
+
+template <typename T>
+bool CompareGradTensorWithValue(const egr::EagerTensor& target, T value) {
+  egr::AutogradMeta* meta = egr::EagerUtils::unsafe_autograd_meta(target);
+  auto grad_dense =
+      std::dynamic_pointer_cast<pten::DenseTensor>(meta->Grad().impl());
+  T* ptr = grad_dense->mutable_data<T>();
+
+  std::vector<T> host_data(grad_dense->numel());
+  if (paddle::platform::is_gpu_place(grad_dense->place())) {
+#ifdef PADDLE_WITH_CUDA
+    paddle::platform::DeviceContextPool& pool =
+        paddle::platform::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<paddle::platform::CUDADeviceContext*>(
+        pool.Get(paddle::platform::CUDAPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(),
+                         paddle::platform::CUDAPlace(), ptr,
+                         sizeof(T) * grad_dense->numel(), stream);
+    ptr = host_data.data();
+#endif
+  }
+  VLOG(6) << "CompareGradTensorWithValue";
+  for (int i = 0; i < grad_dense->numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   paddle::platform::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i, value, ptr[i]));
+  }
+  return true;
+}
+
+template <typename T>
+bool CompareTensorWithValue(const egr::EagerTensor& target, T value) {
+  // TODO(jiabin): Support Selected Rows later
+  auto dense_t = std::dynamic_pointer_cast<pten::DenseTensor>(target.impl());
+  T* ptr = dense_t->mutable_data<T>();
+
+  std::vector<T> host_data(dense_t->numel());
+  if (paddle::platform::is_gpu_place(dense_t->place())) {
+#ifdef PADDLE_WITH_CUDA
+    paddle::platform::DeviceContextPool& pool =
+        paddle::platform::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<paddle::platform::CUDADeviceContext*>(
+        pool.Get(paddle::platform::CUDAPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(),
+                         paddle::platform::CUDAPlace(), ptr,
+                         sizeof(T) * dense_t->numel(), stream);
+    ptr = host_data.data();
+#endif
+  }
+
+  VLOG(6) << "CompareTensorWithValue";
+  for (int i = 0; i < dense_t->numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   paddle::platform::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i, value, ptr[i]));
+  }
+  return true;
+}
+
+template <typename T>
+bool CompareVariableWithValue(const egr::EagerTensor& target, T value) {
+  // TODO(jiabin): Support Selected Rows later
+  auto lod_tensor = target.Var().Get<paddle::framework::LoDTensor>();
+  T* ptr = lod_tensor.data<T>();
+
+  std::vector<T> host_data(lod_tensor.numel());
+  if (paddle::platform::is_gpu_place(lod_tensor.place())) {
+#ifdef PADDLE_WITH_CUDA
+    paddle::platform::DeviceContextPool& pool =
+        paddle::platform::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<paddle::platform::CUDADeviceContext*>(
+        pool.Get(paddle::platform::CUDAPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(),
+                         paddle::platform::CUDAPlace(), ptr,
+                         sizeof(T) * lod_tensor.numel(), stream);
+    ptr = host_data.data();
+#endif
+  }
+  VLOG(6) << "CompareVariableWithValue";
+  for (int i = 0; i < lod_tensor.numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   paddle::platform::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i, value, ptr[i]));
+  }
+  return true;
+}
+
+template <typename T>
+bool CompareGradVariableWithValue(const egr::EagerTensor& target, T value) {
+  // TODO(jiabin): Support Selected Rows later
+  egr::AutogradMeta* meta = egr::EagerUtils::unsafe_autograd_meta(target);
+  auto lod_tensor = meta->Grad().Var().Get<paddle::framework::LoDTensor>();
+  T* ptr = lod_tensor.data<T>();
+
+  std::vector<T> host_data(lod_tensor.numel());
+  if (paddle::platform::is_gpu_place(lod_tensor.place())) {
+#ifdef PADDLE_WITH_CUDA
+    paddle::platform::DeviceContextPool& pool =
+        paddle::platform::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<paddle::platform::CUDADeviceContext*>(
+        pool.Get(paddle::platform::CUDAPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(),
+                         paddle::platform::CUDAPlace(), ptr,
+                         sizeof(T) * lod_tensor.numel(), stream);
+    ptr = host_data.data();
+#endif
+  }
+  VLOG(6) << "CompareGradVariableWithValue";
+  for (int i = 0; i < lod_tensor.numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   paddle::platform::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i, value, ptr[i]));
+  }
+  return true;
+}
+
+inline void InitEnv(paddle::platform::Place place) {
+  // Prepare Device Contexts
+  // Init DeviceContextPool
+  paddle::framework::InitDevices();
+
+  // Init Tracer Place
+  egr::Controller::Instance().SetExpectedPlace(place);
+}
+}  // namespace eager_test

paddle/fluid/eager/utils.cc

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/fluid/eager/api/all.h"
+
+namespace egr {
+/* ---- Tensor -> Var ---- */
+std::vector<std::shared_ptr<egr::EagerTensor>> EagerUtils::SyncToVars(
+    const egr::EagerTensor& tensor) {
+  // TODO(jiabin): No const cast here. We should call SyncToVar in Python_C
+  // wrapper
+  const_cast<EagerTensor*>(&tensor)->SyncToVar(
+      paddle::framework::proto::VarType_Type_LOD_TENSOR);
+  return {std::make_shared<EagerTensor>(tensor)};
+}
+
+std::vector<std::shared_ptr<egr::EagerTensor>> EagerUtils::SyncToVars(
+    const std::vector<egr::EagerTensor>& tensors) {
+  // TODO(jiabin): No const cast here. We should call SyncToVar in Python_C
+  // wrapper
+  std::vector<std::shared_ptr<EagerTensor>> res;
+  size_t num = tensors.size();
+  res.reserve(num);
+  for (size_t i = 0; i < num; i++) {
+    const_cast<EagerTensor*>(&(tensors[i]))
+        ->SyncToVar(paddle::framework::proto::VarType_Type_LOD_TENSOR);
+    res.emplace_back(new EagerTensor(tensors[i]));
+  }
+  return res;
+}
+
+/* ---- VarBase -> Tensor ---- */
+std::vector<std::shared_ptr<egr::EagerTensor>> EagerUtils::SyncToTensors(
+    const egr::EagerTensor& tensor) {
+  // TODO(jiabin): No const cast here. We should call SyncToTensor in Python_C
+  // wrapper
+  const_cast<EagerTensor*>(&tensor)->SyncToTensor();
+  return {std::make_shared<EagerTensor>(tensor)};
+}
+
+std::vector<std::shared_ptr<egr::EagerTensor>> EagerUtils::SyncToTensors(
+    const std::vector<egr::EagerTensor>& tensors) {
+  // TODO(jiabin): No const cast here. We should call SyncToTensor in Python_C
+  // wrapper
+  std::vector<std::shared_ptr<EagerTensor>> res;
+  size_t num = tensors.size();
+  res.reserve(num);
+  for (size_t i = 0; i < num; i++) {
+    const_cast<EagerTensor*>(&(tensors[i]))->SyncToTensor();
+    res.emplace_back(new EagerTensor(tensors[i]));
+  }
+  return res;
+}
+
+std::vector<std::shared_ptr<EagerTensor>> EagerUtils::ConstructDuplicableOutput(
+    const size_t num) {
+  std::vector<std::shared_ptr<EagerTensor>> res;
+  res.reserve(num);
+  for (size_t i = 0; i < num; i++) {
+    res.emplace_back(
+        new EagerTensor(egr::Controller::Instance().GenerateUniqueName()));
+  }
+  return res;
+}
+
+std::vector<egr::EagerTensor> EagerUtils::GetOutputs(
+    const std::vector<std::shared_ptr<EagerTensor>>& outs) {
+  std::vector<egr::EagerTensor> res;
+  res.reserve(outs.size());
+  for (const auto& out : outs) {
+    PADDLE_ENFORCE_NOT_NULL(
+        out.get(), paddle::platform::errors::Fatal(
+                       "Eager Tensor %s is null and cannot be copied. "
+                       "We are tring to Get Output tensor from its "
+                       "shared_ptr, this error may indicate some outputs "
+                       "are nullptr",
+                       out->name()));
+    res.emplace_back((*(out.get())));
+  }
+  return res;
+}
+
+egr::EagerTensor EagerUtils::GetOutput(
+    const std::shared_ptr<EagerTensor>& out) {
+  PADDLE_ENFORCE_NOT_NULL(
+      out.get(), paddle::platform::errors::Fatal(
+                     "Eager Tensor %s is null and cannot be copied. We "
+                     "are tring to Get Output tensor from its shared_ptr, "
+                     "this error may indicate output is nullptr",
+                     out->name()));
+  return EagerTensor((*(out.get())));
+}
+
+AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) {
+  auto* p_autograd_meta = target.get_autograd_meta();
+  PADDLE_ENFORCE(p_autograd_meta,
+                 paddle::platform::errors::Fatal(
+                     "Null autograd_meta gotten from unsafe_autograd_meta(), "
+                     "if you are using unsafe_autograd_meta, please make sure "
+                     "your tensor's autograd_meta is set"));
+  return static_cast<AutogradMeta*>(p_autograd_meta);
+}
+
+std::pair<size_t, size_t> EagerUtils::OutRankInfo(
+    const egr::EagerTensor& target) {
+  return unsafe_autograd_meta(target)->OutRankInfo();
+}
+}  // namespace egr

paddle/fluid/eager/utils.h

+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+
+#include "paddle/pten/api/all.h"
+
+namespace egr {
+
+/**
+ * EagerUtils is utils used to do some static conversion or autograd
+ * members access, this class is desinged to be a full static functional
+ * utils class
+ * **/
+
+template <typename ElementType>
+class IterHelper {
+  virtual void visit(ElementType element) = 0;
+
+  void visit(std::vector<ElementType>* elements) {
+    for (auto element : *elements) visit(element);
+  }
+
+  template <typename... Args>
+  void apply() {}
+
+ public:
+  template <typename T, typename... Args>
+  void apply(T&& arg, Args&&... args) {
+    visit(std::forward<T>(arg));
+    return apply(std::forward<Args>(args)...);
+  }
+  virtual ~IterHelper() = default;
+};
+
+class ComputeRequireGradIter : public IterHelper<AutogradMeta*> {
+ public:
+  bool RequireGrad() { return require_grad_; }
+
+ private:
+  void visit(AutogradMeta* element) override {
+    bool stop_gradient = element->StopGradient();
+    if (!stop_gradient) require_grad_ = true;
+  }
+
+  bool require_grad_ = false;
+};
+
+class PassStopGradientIter : public IterHelper<AutogradMeta*> {
+ public:
+  void SetStopGradient(bool stop_gradient) { stop_gradient_ = stop_gradient; }
+
+ private:
+  void visit(AutogradMeta* element) override {
+    if (!element) {
+      // TODO(jiabin): Add Tensor name here when we supported.
+      VLOG(2) << "Tensor is NULL";
+      return;
+    }
+    element->SetStopGradient(stop_gradient_);
+  }
+
+  bool stop_gradient_ = true;
+};
+
+class EagerUtils {
+ public:
+  /**
+   * We have to use autograd_meta and multi_autograd_meta to initialize
+   * autograd_meta for tensor, since we can't init it in
+   * egr::EagerTensor's
+   * constructor (it's abstract class there)
+   *
+   * **/
+  template <typename T, typename... Args>
+  static bool ComputeRequireGrad(T trace_backward, Args&&... args) {
+    if (!trace_backward) return false;
+
+    auto iter = ComputeRequireGradIter();
+    iter.apply(std::forward<Args>(args)...);
+
+    return iter.RequireGrad();
+  }
+
+  template <typename T, typename... Args>
+  static void PassStopGradient(T stop_gradient, Args&&... args) {
+    auto iter = PassStopGradientIter();
+    iter.SetStopGradient(stop_gradient);
+    iter.apply(std::forward<Args>(args)...);
+  }
+  static std::pair<size_t, size_t> OutRankInfo(const egr::EagerTensor& target);
+  // This method will return an AutogradMeta pointer unsafely.
+  static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target);
+
+  // Intermidate needed remove this once we don't need legacy
+  static std::vector<std::shared_ptr<egr::EagerTensor>> SyncToVars(
+      const egr::EagerTensor& tensor);
+  static std::vector<std::shared_ptr<egr::EagerTensor>> SyncToVars(
+      const std::vector<egr::EagerTensor>& tensors);
+  static std::vector<std::shared_ptr<egr::EagerTensor>> SyncToTensors(
+      const egr::EagerTensor& tensor);
+  static std::vector<std::shared_ptr<egr::EagerTensor>> SyncToTensors(
+      const std::vector<egr::EagerTensor>& tensors);
+  static std::vector<std::shared_ptr<EagerTensor>> ConstructDuplicableOutput(
+      const size_t num);
+  static std::vector<egr::EagerTensor> GetOutputs(
+      const std::vector<std::shared_ptr<EagerTensor>>& outs);
+  static egr::EagerTensor GetOutput(const std::shared_ptr<EagerTensor>& outs);
+};
+
+}  // namespace egr