From 7de99d8ce06460eb04f75f43dfcf3ae959ca2591 Mon Sep 17 00:00:00 2001 From: Zhanlue Yang Date: Wed, 24 Nov 2021 21:06:20 +0800 Subject: [PATCH] Added EagerUtils to Eager Dygraph (#37479) * Added EagerUtils to Eager Dygraph * Purified include dependencies for global_utils * Fixed merge conflicts --- paddle/fluid/eager/CMakeLists.txt | 4 +- paddle/fluid/eager/api/CMakeLists.txt | 2 - paddle/fluid/eager/api/utils/CMakeLists.txt | 2 +- paddle/fluid/eager/api/utils/global_utils.h | 5 +- paddle/fluid/eager/tests/CMakeLists.txt | 2 +- .../eager/tests/task_tests/CMakeLists.txt | 2 +- .../tests/task_tests/eager_utils_test.cc | 87 ++++++++++++++- paddle/fluid/eager/utils.cc | 101 +++++++++++++++--- paddle/fluid/eager/utils.h | 30 +++++- 9 files changed, 205 insertions(+), 30 deletions(-) diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index fe3fe97608..ef5c4bcd73 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -1,5 +1,5 @@ -add_subdirectory(tests) add_subdirectory(api) +add_subdirectory(tests) cc_library(grad_node_info SRCS grad_node_info.cc DEPS pten pten_api) cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api) -cc_library(utils SRCS utils.cc DEPS pten pten_api autograd_meta eager_api) +cc_library(utils SRCS utils.cc DEPS pten pten_api global_utils layer proto_desc operator op_registry variable_helper memcpy scale_op autograd_meta) diff --git a/paddle/fluid/eager/api/CMakeLists.txt b/paddle/fluid/eager/api/CMakeLists.txt index 92c1c81bb8..512d2b1553 100644 --- a/paddle/fluid/eager/api/CMakeLists.txt +++ b/paddle/fluid/eager/api/CMakeLists.txt @@ -1,3 +1 @@ add_subdirectory(utils) - -cc_library(eager_api SRCS all.cc DEPS global_utils) diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index 5168f1fc02..2bce8e0bc3 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1 +1 @@ -cc_library(global_utils SRCS global_utils.cc DEPS enforce) +cc_library(global_utils SRCS global_utils.cc DEPS place) diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h index 16e7ef8a58..39db269a02 100644 --- a/paddle/fluid/eager/api/utils/global_utils.h +++ b/paddle/fluid/eager/api/utils/global_utils.h @@ -15,8 +15,9 @@ #pragma once -#include "paddle/fluid/eager/eager_tensor.h" -#include "paddle/fluid/platform/enforce.h" +#include +#include +#include "paddle/fluid/platform/place.h" namespace egr { diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index bdf542f20e..35b9934141 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,3 +1,3 @@ -set(eager_deps pten pten_api) +set(eager_deps pten pten_api pten_tensor utils global_utils autograd_meta grad_node_info) add_subdirectory(data_structure_tests) add_subdirectory(task_tests) diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt index 61f89d9452..6ab5c70b44 100644 --- a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -1 +1 @@ -cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps} grad_node_info autograd_meta utils) +cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps}) diff --git a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc index 3df0a77aed..4d93f0188a 100644 --- a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc +++ b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc @@ -12,17 +12,96 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Eager Dygraph +#include #include "gtest/gtest.h" -#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/eager_tensor.h" #include "paddle/fluid/eager/grad_node_info.h" #include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h" #include "paddle/fluid/eager/tests/test_utils.h" #include "paddle/fluid/eager/utils.h" +#include "paddle/pten/api/lib/utils/allocator.h" + +// TODO(jiabin): remove nolint here!!! +using namespace egr; // NOLINT + namespace eager_test { + +TEST(EagerUtils, AutoGradMeta) { + // Construct Eager Tensor + pten::DenseTensorMeta meta = pten::DenseTensorMeta( + pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 1})); + std::shared_ptr dt0 = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + dt0->mutable_data()[0] = 10.0; + EagerTensor et0 = EagerTensor(dt0); + + std::shared_ptr dt1 = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + dt1->mutable_data()[0] = 20.0; + EagerTensor et1 = EagerTensor(dt1); + + std::vector ets = {et0, et1}; + auto test_node = std::make_shared(); + + // unsafe_autograd_meta() + // autograd_meta() + // multi_autograd_meta() + AutogradMeta* autograd_meta0 = EagerUtils::autograd_meta(&et0); + AutogradMeta* autograd_meta1 = EagerUtils::autograd_meta(&et1); + + AutogradMeta* unsafe_autograd_meta_after = + EagerUtils::unsafe_autograd_meta(et0); + CHECK_NOTNULL(unsafe_autograd_meta_after); + + std::vector autograd_metas = + EagerUtils::multi_autograd_meta(&ets); + std::vector unsafe_autograd_metas = + EagerUtils::unsafe_autograd_meta(&ets); + CHECK_NOTNULL(unsafe_autograd_metas[0]); + CHECK_NOTNULL(unsafe_autograd_metas[1]); + + // Set Autograd Meta + autograd_meta0->SetSingleOutRankWithSlot(0, 1); + + autograd_meta0->SetGradNode(test_node); + + // OutRankInfo() + std::pair out_rank_info0 = EagerUtils::OutRankInfo(et0); + CHECK_EQ(static_cast(out_rank_info0.first), 0); + CHECK_EQ(static_cast(out_rank_info0.second), 1); + + // grad_node() + std::shared_ptr grad_node0 = EagerUtils::grad_node(et0); + CHECK_NOTNULL(grad_node0.get()); + + EagerUtils::SetHistory(autograd_meta1, test_node); + EagerUtils::SetHistory({autograd_meta1}, test_node); + std::shared_ptr grad_node1 = EagerUtils::grad_node(et1); + CHECK_NOTNULL(grad_node1.get()); + + // SetOutRankWithSlot() + EagerUtils::SetOutRankWithSlot(autograd_meta1, 0); + std::pair out_rank_info1 = EagerUtils::OutRankInfo(et1); + CHECK_EQ(static_cast(out_rank_info1.first), 0); + CHECK_EQ(static_cast(out_rank_info1.second), 0); + + EagerUtils::SetOutRankWithSlot(&autograd_metas, 0); + std::pair out_rank_info2 = EagerUtils::OutRankInfo(et0); + CHECK_EQ(static_cast(out_rank_info2.first), 0); + CHECK_EQ(static_cast(out_rank_info2.second), 0); + + std::pair out_rank_info3 = EagerUtils::OutRankInfo(et1); + CHECK_EQ(static_cast(out_rank_info3.first), 0); + CHECK_EQ(static_cast(out_rank_info3.second), 1); +} + template egr::EagerTensor CreateTestCPUTensor(T val, const paddle::framework::DDim& ddim) { @@ -40,7 +119,7 @@ egr::EagerTensor CreateTestCPUTensor(T val, tensor.set_impl(dt); return tensor; } -} // namespace eager_test + TEST(EagerUtils, ComputeRequireGrad) { auto auto_grad0 = std::make_shared(); auto auto_grad1 = std::make_shared(); @@ -200,3 +279,5 @@ TEST(EagerUtils, ConstructDuplicableOutput) { CHECK(outs[0]->defined() == false); CHECK(outs[0]->initialized() == false); } + +} // namespace eager_test diff --git a/paddle/fluid/eager/utils.cc b/paddle/fluid/eager/utils.cc index c5bda181d4..0b7e6e357d 100644 --- a/paddle/fluid/eager/utils.cc +++ b/paddle/fluid/eager/utils.cc @@ -13,9 +13,94 @@ // limitations under the License. #include "paddle/fluid/eager/utils.h" -#include "paddle/fluid/eager/api/all.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" + +#include "paddle/pten/api/all.h" +#include "paddle/pten/common/layout.h" +#include "paddle/pten/core/tensor_meta.h" + +#include "paddle/fluid/framework/data_layout.h" +#include "paddle/fluid/framework/pten_utils.h" +#include "paddle/fluid/framework/variable.h" namespace egr { +/** + * Implementation of Eager Utils. +**/ + +AutogradMeta* EagerUtils::autograd_meta(egr::EagerTensor* target) { + auto* p_autograd_meta = target->get_autograd_meta(); + if (!p_autograd_meta) { + auto p_autograd_meta_ptr = std::make_shared(); + p_autograd_meta = p_autograd_meta_ptr.get(); + target->set_autograd_meta(p_autograd_meta_ptr); + } + return static_cast(p_autograd_meta); +} + +AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) { + auto* p_autograd_meta = target.get_autograd_meta(); + PADDLE_ENFORCE(p_autograd_meta, + paddle::platform::errors::Fatal( + "Null autograd_meta gotten from unsafe_autograd_meta()")); + return static_cast(p_autograd_meta); +} + +std::vector EagerUtils::unsafe_autograd_meta( + std::vector* targets) { + std::vector metas; + for (const egr::EagerTensor& t : *targets) { + metas.push_back(unsafe_autograd_meta(t)); + } + return metas; +} + +std::vector EagerUtils::multi_autograd_meta( + std::vector* targets) { + std::vector ret; + ret.reserve(targets->size()); + + // for multi_autograd_meta we can tolerent it has nullptr. + for (auto& t : (*targets)) { + auto* p_autograd_meta = autograd_meta(&t); + ret.push_back(static_cast(p_autograd_meta)); + } + return ret; +} + +std::pair EagerUtils::OutRankInfo( + const egr::EagerTensor& target) { + return unsafe_autograd_meta(target)->OutRankInfo(); +} + +std::shared_ptr EagerUtils::grad_node( + const egr::EagerTensor& target) { + return unsafe_autograd_meta(target)->GetMutableGradNode(); +} + +void EagerUtils::SetHistory(std::vector* autograd_metas, + const std::shared_ptr& grad_node) { + for (const auto& autograd_meta : *autograd_metas) { + autograd_meta->SetGradNode(grad_node); + } +} + +void EagerUtils::SetHistory(AutogradMeta* autograd_meta, + const std::shared_ptr& grad_node) { + autograd_meta->SetGradNode(grad_node); +} + +void EagerUtils::SetOutRankWithSlot(std::vector* targets, + size_t slot_id) { + // Set OutRankInfo from 0 to size of targets + for (size_t i = 0; i < targets->size(); i++) { + (*targets)[i]->SetSingleOutRankWithSlot(slot_id, i); + } +} +void EagerUtils::SetOutRankWithSlot(AutogradMeta* target, size_t slot_id) { + target->SetSingleOutRankWithSlot(slot_id, 0); +} + /* ---- Tensor -> Var ---- */ std::vector> EagerUtils::SyncToVars( const egr::EagerTensor& tensor) { @@ -103,18 +188,4 @@ egr::EagerTensor EagerUtils::GetOutput( return EagerTensor((*(out.get()))); } -AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) { - auto* p_autograd_meta = target.get_autograd_meta(); - PADDLE_ENFORCE(p_autograd_meta, - paddle::platform::errors::Fatal( - "Null autograd_meta gotten from unsafe_autograd_meta(), " - "if you are using unsafe_autograd_meta, please make sure " - "your tensor's autograd_meta is set")); - return static_cast(p_autograd_meta); -} - -std::pair EagerUtils::OutRankInfo( - const egr::EagerTensor& target) { - return unsafe_autograd_meta(target)->OutRankInfo(); -} } // namespace egr diff --git a/paddle/fluid/eager/utils.h b/paddle/fluid/eager/utils.h index 4e8461e660..19692df78d 100644 --- a/paddle/fluid/eager/utils.h +++ b/paddle/fluid/eager/utils.h @@ -87,6 +87,33 @@ class EagerUtils { * constructor (it's abstract class there) * * **/ + static AutogradMeta* autograd_meta(egr::EagerTensor* target); + + static std::vector multi_autograd_meta( + std::vector* targets); + + static std::pair OutRankInfo(const egr::EagerTensor& target); + + static std::shared_ptr grad_node( + const egr::EagerTensor& target); + + // Set history is used to set backward info during forward process, it will + // set forward var's autograd meta's grad node as current backward node. + static void SetHistory(std::vector* autograd_metas, + const std::shared_ptr& grad_node); + static void SetHistory(AutogradMeta* autograd_meta, + const std::shared_ptr& grad_node); + + // This is used for Set vector of tensors' rank + static void SetOutRankWithSlot(std::vector* targets, + size_t slot_id); + static void SetOutRankWithSlot(AutogradMeta* target, size_t slot_id); + + // This method will return an AutogradMeta pointer unsafely. + static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target); + static std::vector unsafe_autograd_meta( + std::vector* targets); + template static bool ComputeRequireGrad(T trace_backward, Args&&... args) { if (!trace_backward) return false; @@ -103,9 +130,6 @@ class EagerUtils { iter.SetStopGradient(stop_gradient); iter.apply(std::forward(args)...); } - static std::pair OutRankInfo(const egr::EagerTensor& target); - // This method will return an AutogradMeta pointer unsafely. - static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target); // Intermidate needed remove this once we don't need legacy static std::vector> SyncToVars( -- GitLab