From 1799c0322795d76dfae6d6cf6d62996d92974dbd Mon Sep 17 00:00:00 2001 From: Jiabin Yang Date: Wed, 24 Nov 2021 10:38:23 +0800 Subject: [PATCH] Refactor dygraph to eager -- TensorWrapper, EagerUtils, GlobalUtils (#37466) * Add EagerTensor and tests * remove useless enforce * remove comment in cmake * support autograd meta * support grad node info test * support grad_node_info * add more edge test * remove Python.h * add tensor wrapper with tests * support compute require grad and stop gradient * support sync methods and global utils * support pure cpu test * refine error msg * refine error msg * refine error info * fix npu error --- paddle/fluid/eager/CMakeLists.txt | 2 + paddle/fluid/eager/api/CMakeLists.txt | 3 + paddle/fluid/eager/api/all.cc | 18 ++ paddle/fluid/eager/api/all.h | 17 ++ paddle/fluid/eager/api/utils/CMakeLists.txt | 1 + paddle/fluid/eager/api/utils/global_utils.cc | 22 ++ paddle/fluid/eager/api/utils/global_utils.h | 62 ++++++ paddle/fluid/eager/eager_tensor.h | 1 - paddle/fluid/eager/tensor_wrapper.h | 91 ++++++++ paddle/fluid/eager/tests/CMakeLists.txt | 1 + .../tests/data_structure_tests/CMakeLists.txt | 1 + .../data_structure_tests/grad_node_test.h | 3 + .../tensor_wrapper_test.cc | 80 +++++++ .../eager/tests/task_tests/CMakeLists.txt | 1 + .../tests/task_tests/eager_utils_test.cc | 202 ++++++++++++++++++ paddle/fluid/eager/tests/test_utils.h | 175 +++++++++++++++ paddle/fluid/eager/utils.cc | 120 +++++++++++ paddle/fluid/eager/utils.h | 126 +++++++++++ 18 files changed, 925 insertions(+), 1 deletion(-) create mode 100644 paddle/fluid/eager/api/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/all.cc create mode 100644 paddle/fluid/eager/api/all.h create mode 100644 paddle/fluid/eager/api/utils/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/utils/global_utils.cc create mode 100644 paddle/fluid/eager/api/utils/global_utils.h create mode 100644 paddle/fluid/eager/tensor_wrapper.h create mode 100644 paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc create mode 100644 paddle/fluid/eager/tests/task_tests/CMakeLists.txt create mode 100644 paddle/fluid/eager/tests/task_tests/eager_utils_test.cc create mode 100644 paddle/fluid/eager/tests/test_utils.h create mode 100644 paddle/fluid/eager/utils.cc create mode 100644 paddle/fluid/eager/utils.h diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index a79b451b544..fe3fe976085 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -1,3 +1,5 @@ add_subdirectory(tests) +add_subdirectory(api) cc_library(grad_node_info SRCS grad_node_info.cc DEPS pten pten_api) cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api) +cc_library(utils SRCS utils.cc DEPS pten pten_api autograd_meta eager_api) diff --git a/paddle/fluid/eager/api/CMakeLists.txt b/paddle/fluid/eager/api/CMakeLists.txt new file mode 100644 index 00000000000..92c1c81bb8c --- /dev/null +++ b/paddle/fluid/eager/api/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(utils) + +cc_library(eager_api SRCS all.cc DEPS global_utils) diff --git a/paddle/fluid/eager/api/all.cc b/paddle/fluid/eager/api/all.cc new file mode 100644 index 00000000000..2308e934177 --- /dev/null +++ b/paddle/fluid/eager/api/all.cc @@ -0,0 +1,18 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "paddle/fluid/eager/api/all.h" + +namespace egr {} // namespace egr diff --git a/paddle/fluid/eager/api/all.h b/paddle/fluid/eager/api/all.h new file mode 100644 index 00000000000..4d873ad95a4 --- /dev/null +++ b/paddle/fluid/eager/api/all.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once + +#include "paddle/fluid/eager/api/utils/global_utils.h" diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt new file mode 100644 index 00000000000..5168f1fc024 --- /dev/null +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(global_utils SRCS global_utils.cc DEPS enforce) diff --git a/paddle/fluid/eager/api/utils/global_utils.cc b/paddle/fluid/eager/api/utils/global_utils.cc new file mode 100644 index 00000000000..3a6a05eb1bf --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "paddle/fluid/eager/api/utils/global_utils.h" + +namespace egr { + +Controller* Controller::controller_ = new Controller(); + +} // namespace egr diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h new file mode 100644 index 00000000000..16e7ef8a58e --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.h @@ -0,0 +1,62 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/platform/enforce.h" + +namespace egr { + +class UniqueNameGenerator { + public: + explicit UniqueNameGenerator(std::string prefix = "") : prefix_(prefix) {} + std::string Generate(std::string key = "eager_tmp") { + return prefix_ + key + "_" + std::to_string(id_++); + } + + private: + std::atomic id_{0}; + std::string prefix_; +}; + +// Global +class Controller { + public: + static Controller& Instance() { return *controller_; } + const paddle::platform::Place& GetExpectedPlace() const { + return *expected_place_.get(); + } + void SetExpectedPlace(const paddle::platform::Place& place) { + expected_place_ = std::make_shared(place); + } + void SetAMPLevel(int level) { amp_level_ = level; } + int GetAMPLevel() const { return amp_level_; } + bool HasGrad() const { return has_grad_; } + std::string GenerateUniqueName(std::string key = "eager_tmp") { + return generator_->Generate(key); + } + + private: + Controller() = default; + static Controller* controller_; + std::shared_ptr expected_place_ = nullptr; + int amp_level_ = 0; + bool has_grad_ = true; + std::unique_ptr generator_{new UniqueNameGenerator()}; + DISABLE_COPY_AND_ASSIGN(Controller); +}; + +} // namespace egr diff --git a/paddle/fluid/eager/eager_tensor.h b/paddle/fluid/eager/eager_tensor.h index d871a84dc22..753040a2623 100644 --- a/paddle/fluid/eager/eager_tensor.h +++ b/paddle/fluid/eager/eager_tensor.h @@ -14,7 +14,6 @@ #pragma once // framework deps -#include "paddle/fluid/framework/data_layout_transform.h" #include "paddle/fluid/framework/pten_utils.h" #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/framework/variable.h" diff --git a/paddle/fluid/eager/tensor_wrapper.h b/paddle/fluid/eager/tensor_wrapper.h new file mode 100644 index 00000000000..d760a76ec68 --- /dev/null +++ b/paddle/fluid/eager/tensor_wrapper.h @@ -0,0 +1,91 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * We now still need TensorWrapper and it is designed to Copy + * tensor in autograd mode. + * + * Since in autograd usage, we need to pass autograd_meta to + * backward computation however in tensor interface add to much + * autograd_related method is not a good choice. + * + * In TensorWrapper we will keep autograd info to backward, only + * for input var, but for output var it will only copy autograd + * with no grad **/ + +#pragma once +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/utils.h" + +namespace egr { +class TensorWrapper { + public: + TensorWrapper() = default; + explicit TensorWrapper(const egr::EagerTensor& tensor, + bool full_reserved = false) { + /** + * Normally, we should fully reserved all non-output or non-leaf fwd tensor + * here. And for fwd output tensor, we should not reserve its autogradmeta, + * to avoid recursive depends on GradNodeBase + * **/ + full_reserved_ = full_reserved; + if (full_reserved_) { + VLOG(6) << "Fully reserved tensor: " << tensor.name(); + intermidiate_tensor_ = tensor; + return; + } + + // shallow copy tensor_impl here + intermidiate_tensor_.set_impl(tensor.impl()); + intermidiate_tensor_.ResetVar(tensor.Var()); + intermidiate_tensor_.set_name(tensor.name() + "@Saved"); + PADDLE_ENFORCE_NOT_NULL( + EagerUtils::unsafe_autograd_meta(tensor), + paddle::platform::errors::Fatal( + "Full reserved Tensor should not have null autograd meta, since " + "tensor_wrapper is used to build backward info. There is no way " + "for us to build it with null autograd_meta.")); + // copy output_rank + out_rank_info_ = EagerUtils::OutRankInfo(tensor); + } + + egr::EagerTensor recover(const std::shared_ptr& grad_node) { + VLOG(6) << "Recover tensor for wrapper"; + if ((!intermidiate_tensor_.defined()) && + (!intermidiate_tensor_.Var().IsInitialized())) { + VLOG(6) << "Return NULL tensor Here. "; + return egr::EagerTensor(); + } + + // if it's full_reserved just return the full copy of tensor + if (full_reserved_) { + return intermidiate_tensor_; + } else { + std::shared_ptr new_grad_node = grad_node; + auto p_ab_autograd_meta = + std::make_shared(Edge(new_grad_node, out_rank_info_)); + intermidiate_tensor_.set_autograd_meta( + std::static_pointer_cast( + p_ab_autograd_meta)); + return intermidiate_tensor_; + } + } + + private: + bool full_reserved_ = false; + std::pair out_rank_info_; + egr::EagerTensor intermidiate_tensor_; +}; +} // namespace egr diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index 572740e03c6..bdf542f20e0 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,2 +1,3 @@ set(eager_deps pten pten_api) add_subdirectory(data_structure_tests) +add_subdirectory(task_tests) diff --git a/paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt b/paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt index 21e63b6480c..2989330efa8 100644 --- a/paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt @@ -1,3 +1,4 @@ cc_test(test_egr_ds_eager_tensor SRCS eager_tensor_test.cc DEPS ${eager_deps} ) cc_test(test_egr_ds_auotgrad_meta SRCS autograd_meta_test.cc DEPS ${eager_deps} grad_node_info) cc_test(test_egr_ds_grad_node_info SRCS grad_node_info_test.cc DEPS ${eager_deps} grad_node_info) +cc_test(test_egr_ds_tensor_wrapper SRCS tensor_wrapper_test.cc DEPS ${eager_deps} grad_node_info utils) diff --git a/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h b/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h index ddea70da791..2870bfa8b0c 100644 --- a/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h +++ b/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h @@ -19,6 +19,9 @@ #include "paddle/fluid/eager/eager_tensor.h" #include "paddle/fluid/eager/grad_node_info.h" #include "paddle/pten/api/lib/utils/allocator.h" +namespace egr { +class TensorWrapper; +} namespace eager_test { class GradTestNode : public egr::GradNodeBase { diff --git a/paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc b/paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc new file mode 100644 index 00000000000..6d78cf42d0c --- /dev/null +++ b/paddle/fluid/eager/tests/data_structure_tests/tensor_wrapper_test.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glog/logging.h" +#include "gtest/gtest.h" + +#include "paddle/fluid/eager/tensor_wrapper.h" +#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h" +#include "paddle/fluid/eager/utils.h" + +TEST(TensorWrapper, Basic) { + VLOG(6) << "Test Full reserved"; + egr::EagerTensor et1; + pten::DenseTensorMeta meta = pten::DenseTensorMeta( + pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 2})); + std::shared_ptr dt = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + auto* dt_ptr = dt->mutable_data(); + dt_ptr[0] = 5.0f; + dt_ptr[1] = 10.0f; + et1.set_impl(dt); + // Create grad node; + auto grad_test_node0 = std::make_shared( + /* val */ 5.0, /* in_num */ 2, /* out_num */ 2); + egr::Edge edge0(grad_test_node0, 1, 2); + auto auto_grad0 = std::make_shared(edge0); + et1.set_autograd_meta(auto_grad0); + et1.set_name("et1"); + auto tw0 = egr::TensorWrapper(et1, true); + auto recover_et1 = tw0.recover(std::make_shared()); + CHECK_EQ(recover_et1.name(), std::string("et1")); + CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et1).first, + egr::EagerUtils::OutRankInfo(et1).first); + CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et1).second, + egr::EagerUtils::OutRankInfo(et1).second); + VLOG(6) << "Test reconstruct"; + egr::EagerTensor et2; + pten::DenseTensorMeta meta2 = pten::DenseTensorMeta( + pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 2})); + std::shared_ptr dt2 = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta2); + auto* dt_ptr2 = dt->mutable_data(); + dt_ptr2[0] = 6.0f; + dt_ptr2[1] = 11.0f; + et2.set_impl(dt2); + et2.set_name("et2"); + auto grad_test_node1 = + std::make_shared(/* val */ 5.0, 2, 2); + egr::Edge edge1(grad_test_node1, 1, 2); + auto auto_grad1 = std::make_shared(edge1); + et2.set_autograd_meta(auto_grad1); + auto tw1 = egr::TensorWrapper(et2, false); + auto recover_et2 = tw1.recover(grad_test_node1); + CHECK_EQ(recover_et2.name(), std::string("et2@Saved")); + CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et2).first, + egr::EagerUtils::OutRankInfo(et2).first); + CHECK_EQ(egr::EagerUtils::OutRankInfo(recover_et2).second, + egr::EagerUtils::OutRankInfo(et2).second); + // Test Raw recover + egr::EagerTensor et3; + auto tw2 = egr::TensorWrapper(et3, true); + CHECK( + tw2.recover(std::make_shared()).initialized() == + false); +} diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt new file mode 100644 index 00000000000..61f89d94525 --- /dev/null +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -0,0 +1 @@ +cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps} grad_node_info autograd_meta utils) diff --git a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc new file mode 100644 index 00000000000..3df0a77aed0 --- /dev/null +++ b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc @@ -0,0 +1,202 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Eager Dygraph + +#include "gtest/gtest.h" + +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h" +#include "paddle/fluid/eager/tests/test_utils.h" +#include "paddle/fluid/eager/utils.h" + +namespace eager_test { +template +egr::EagerTensor CreateTestCPUTensor(T val, + const paddle::framework::DDim& ddim) { + pten::DenseTensorMeta meta = + pten::DenseTensorMeta(pten::DataType::FLOAT32, ddim); + egr::EagerTensor tensor; + std::shared_ptr dt = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + auto* dt_ptr = dt->mutable_data(); + for (int64_t i = 0; i < dt->numel(); i++) { + dt_ptr[i] = val; + } + tensor.set_impl(dt); + return tensor; +} +} // namespace eager_test +TEST(EagerUtils, ComputeRequireGrad) { + auto auto_grad0 = std::make_shared(); + auto auto_grad1 = std::make_shared(); + auto auto_grad2 = std::make_shared(); + auto auto_grad3 = std::make_shared(); + CHECK_EQ(auto_grad0->NumericStopGradient(), -1); + VLOG(6) << "Single Test ComputeRequireGrad"; + auto_grad0->SetStopGradient(true); + CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()) == false); + CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()) == false); + auto_grad0->SetStopGradient(false); + CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()) == false); + CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()) == true); + + VLOG(6) << "Multi Test ComputeRequireGrad"; + auto_grad0->SetStopGradient(false); + auto_grad1->SetStopGradient(true); + CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get(), + auto_grad1.get()) == true); + CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get(), + auto_grad1.get()) == false); + auto_grad0->SetStopGradient(true); + CHECK(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get(), + auto_grad1.get()) == false); + CHECK(egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get(), + auto_grad1.get()) == false); +} + +TEST(EagerUtils, PassStopGradient) { + auto auto_grad0 = std::make_shared(); + auto auto_grad1 = std::make_shared(); + auto auto_grad2 = std::make_shared(); + auto auto_grad3 = std::make_shared(); + CHECK_EQ(auto_grad0->NumericStopGradient(), -1); + VLOG(6) << "Test PassStopGradient"; + egr::EagerUtils::PassStopGradient(false, auto_grad0.get()); + CHECK(auto_grad0->StopGradient() == false); + egr::EagerUtils::PassStopGradient(true, auto_grad0.get(), auto_grad1.get(), + auto_grad2.get(), auto_grad3.get()); + CHECK(auto_grad0->StopGradient() == true); + CHECK(auto_grad1->StopGradient() == true); + CHECK(auto_grad2->StopGradient() == true); + CHECK(auto_grad3->StopGradient() == true); +} + +TEST(EagerUtils, SyncToVarsSingle) { + paddle::framework::DDim ddim = paddle::framework::make_ddim({2, 4, 4, 4}); + auto tensor = eager_test::CreateTestCPUTensor(5.0f, ddim); + std::vector> var_bases = + egr::EagerUtils::SyncToVars(tensor); + + paddle::framework::Variable* var = var_bases[0]->MutableVar(); + const auto& framework_tensor = var->Get(); + + const float* ptr = framework_tensor.data(); + VLOG(6) << "Check Value for SyncToVarsSingle"; + CHECK_EQ(framework_tensor.numel(), tensor.numel()); + + for (int i = 0; i < framework_tensor.numel(); i++) { + CHECK_EQ(ptr[i], 5.0f); + } +} + +TEST(EagerUtils, SyncToVarsMultiple) { + paddle::framework::DDim ddim = paddle::framework::make_ddim({2, 4, 4, 4}); + std::vector tensors = { + eager_test::CreateTestCPUTensor(1.0f, ddim), + eager_test::CreateTestCPUTensor(2.0f, ddim)}; + + std::vector> var_bases = + egr::EagerUtils::SyncToVars(tensors); + + { + paddle::framework::Variable* var = var_bases[0]->MutableVar(); + const auto& framework_tensor = var->Get(); + + const float* ptr = framework_tensor.data(); + CHECK_EQ(framework_tensor.numel(), tensors[0].numel()); + + for (int i = 0; i < framework_tensor.numel(); i++) { + CHECK_EQ(ptr[i], 1.0); + } + } + + { + paddle::framework::Variable* var = var_bases[1]->MutableVar(); + const auto& framework_tensor = var->Get(); + + const float* ptr = framework_tensor.data(); + VLOG(6) << "Check Value for SyncToVarsMultiple"; + CHECK_EQ(framework_tensor.numel(), tensors[0].numel()); + + for (int i = 0; i < framework_tensor.numel(); i++) { + CHECK_EQ(ptr[i], 2.0); + } + } +} + +TEST(EagerUtils, SyncToTensorSingle) { + std::shared_ptr X(new egr::EagerTensor()); + std::vector src_data(128, 5.0); + std::vector dims = {2, 4, 4, 4}; + paddle::platform::CPUPlace place; + + auto* x_tensor = X->MutableVar()->GetMutable(); + x_tensor->Resize(paddle::framework::make_ddim(dims)); + auto* mutable_x = x_tensor->mutable_data(place); + paddle::memory::Copy(place, mutable_x, place, src_data.data(), + sizeof(float) * src_data.size()); + auto X_ = egr::EagerUtils::SyncToTensors(*(X.get())); + egr::EagerTensor tensor = egr::EagerUtils::GetOutput(X_[0]); + VLOG(6) << "Check Value for SyncToTensorSingle"; + CHECK(eager_test::CompareTensorWithValue(tensor, 5.0)); +} + +TEST(EagerUtils, SyncToTensorMultiple) { + eager_test::InitEnv(paddle::platform::CPUPlace()); + std::vector dims = {2, 4, 4, 4}; + paddle::platform::CPUPlace place; + + std::vector egr_tensors; + { + auto egr_tensor = egr::EagerTensor(); + std::vector src_data(128, 1.0); + auto* x_tensor = + egr_tensor.MutableVar()->GetMutable(); + x_tensor->Resize(paddle::framework::make_ddim(dims)); + auto* mutable_x = x_tensor->mutable_data(place); + paddle::memory::Copy(place, mutable_x, place, src_data.data(), + sizeof(float) * src_data.size()); + egr_tensors.emplace_back(egr_tensor); + } + { + auto egr_tensor = egr::EagerTensor(); + std::vector src_data(128, 2.0); + auto* x_tensor = + egr_tensor.MutableVar()->GetMutable(); + x_tensor->Resize(paddle::framework::make_ddim(dims)); + auto* mutable_x = x_tensor->mutable_data(place); + paddle::memory::Copy(place, mutable_x, place, src_data.data(), + sizeof(float) * src_data.size()); + egr_tensors.emplace_back(std::move(egr_tensor)); + } + std::vector tensors = + egr::EagerUtils::GetOutputs(egr::EagerUtils::SyncToTensors(egr_tensors)); + + VLOG(6) << "Check Value for SyncToTensorMultiple"; + CHECK(eager_test::CompareTensorWithValue(tensors[0], 1.0) == true); + CHECK(eager_test::CompareTensorWithValue(tensors[1], 2.0) == true); +} + +TEST(EagerUtils, ConstructDuplicableOutput) { + VLOG(6) << "Check ConstructDuplicableOutput"; + std::vector> outs = + egr::EagerUtils::ConstructDuplicableOutput(2); + CHECK_EQ(outs.size(), size_t(2)); + CHECK(outs[0]->defined() == false); + CHECK(outs[0]->initialized() == false); +} diff --git a/paddle/fluid/eager/tests/test_utils.h b/paddle/fluid/eager/tests/test_utils.h new file mode 100644 index 00000000000..b98ff72f0f0 --- /dev/null +++ b/paddle/fluid/eager/tests/test_utils.h @@ -0,0 +1,175 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/api/utils/global_utils.h" +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/utils.h" + +#include "paddle/pten/api/all.h" +#include "paddle/pten/core/dense_tensor.h" +#include "paddle/pten/core/tensor_meta.h" + +#include "paddle/fluid/memory/memcpy.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/init.h" + +namespace eager_test { + +template +bool CompareGradTensorWithValue(const egr::EagerTensor& target, T value) { + egr::AutogradMeta* meta = egr::EagerUtils::unsafe_autograd_meta(target); + auto grad_dense = + std::dynamic_pointer_cast(meta->Grad().impl()); + T* ptr = grad_dense->mutable_data(); + + std::vector host_data(grad_dense->numel()); + if (paddle::platform::is_gpu_place(grad_dense->place())) { +#ifdef PADDLE_WITH_CUDA + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + auto* dev_ctx = dynamic_cast( + pool.Get(paddle::platform::CUDAPlace())); + auto stream = dev_ctx->stream(); + + paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(), + paddle::platform::CUDAPlace(), ptr, + sizeof(T) * grad_dense->numel(), stream); + ptr = host_data.data(); +#endif + } + VLOG(6) << "CompareGradTensorWithValue"; + for (int i = 0; i < grad_dense->numel(); i++) { + PADDLE_ENFORCE(value == ptr[i], + paddle::platform::errors::PreconditionNotMet( + "Numerical Error in Compare Grad Variable With Value of " + "%d, we expected got value: %f, but got: %f instead. " + "Please check it later.", + i, value, ptr[i])); + } + return true; +} + +template +bool CompareTensorWithValue(const egr::EagerTensor& target, T value) { + // TODO(jiabin): Support Selected Rows later + auto dense_t = std::dynamic_pointer_cast(target.impl()); + T* ptr = dense_t->mutable_data(); + + std::vector host_data(dense_t->numel()); + if (paddle::platform::is_gpu_place(dense_t->place())) { +#ifdef PADDLE_WITH_CUDA + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + auto* dev_ctx = dynamic_cast( + pool.Get(paddle::platform::CUDAPlace())); + auto stream = dev_ctx->stream(); + + paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(), + paddle::platform::CUDAPlace(), ptr, + sizeof(T) * dense_t->numel(), stream); + ptr = host_data.data(); +#endif + } + + VLOG(6) << "CompareTensorWithValue"; + for (int i = 0; i < dense_t->numel(); i++) { + PADDLE_ENFORCE(value == ptr[i], + paddle::platform::errors::PreconditionNotMet( + "Numerical Error in Compare Grad Variable With Value of " + "%d, we expected got value: %f, but got: %f instead. " + "Please check it later.", + i, value, ptr[i])); + } + return true; +} + +template +bool CompareVariableWithValue(const egr::EagerTensor& target, T value) { + // TODO(jiabin): Support Selected Rows later + auto lod_tensor = target.Var().Get(); + T* ptr = lod_tensor.data(); + + std::vector host_data(lod_tensor.numel()); + if (paddle::platform::is_gpu_place(lod_tensor.place())) { +#ifdef PADDLE_WITH_CUDA + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + auto* dev_ctx = dynamic_cast( + pool.Get(paddle::platform::CUDAPlace())); + auto stream = dev_ctx->stream(); + + paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(), + paddle::platform::CUDAPlace(), ptr, + sizeof(T) * lod_tensor.numel(), stream); + ptr = host_data.data(); +#endif + } + VLOG(6) << "CompareVariableWithValue"; + for (int i = 0; i < lod_tensor.numel(); i++) { + PADDLE_ENFORCE(value == ptr[i], + paddle::platform::errors::PreconditionNotMet( + "Numerical Error in Compare Grad Variable With Value of " + "%d, we expected got value: %f, but got: %f instead. " + "Please check it later.", + i, value, ptr[i])); + } + return true; +} + +template +bool CompareGradVariableWithValue(const egr::EagerTensor& target, T value) { + // TODO(jiabin): Support Selected Rows later + egr::AutogradMeta* meta = egr::EagerUtils::unsafe_autograd_meta(target); + auto lod_tensor = meta->Grad().Var().Get(); + T* ptr = lod_tensor.data(); + + std::vector host_data(lod_tensor.numel()); + if (paddle::platform::is_gpu_place(lod_tensor.place())) { +#ifdef PADDLE_WITH_CUDA + paddle::platform::DeviceContextPool& pool = + paddle::platform::DeviceContextPool::Instance(); + auto* dev_ctx = dynamic_cast( + pool.Get(paddle::platform::CUDAPlace())); + auto stream = dev_ctx->stream(); + + paddle::memory::Copy(paddle::platform::CPUPlace(), host_data.data(), + paddle::platform::CUDAPlace(), ptr, + sizeof(T) * lod_tensor.numel(), stream); + ptr = host_data.data(); +#endif + } + VLOG(6) << "CompareGradVariableWithValue"; + for (int i = 0; i < lod_tensor.numel(); i++) { + PADDLE_ENFORCE(value == ptr[i], + paddle::platform::errors::PreconditionNotMet( + "Numerical Error in Compare Grad Variable With Value of " + "%d, we expected got value: %f, but got: %f instead. " + "Please check it later.", + i, value, ptr[i])); + } + return true; +} + +inline void InitEnv(paddle::platform::Place place) { + // Prepare Device Contexts + // Init DeviceContextPool + paddle::framework::InitDevices(); + + // Init Tracer Place + egr::Controller::Instance().SetExpectedPlace(place); +} +} // namespace eager_test diff --git a/paddle/fluid/eager/utils.cc b/paddle/fluid/eager/utils.cc new file mode 100644 index 00000000000..c5bda181d40 --- /dev/null +++ b/paddle/fluid/eager/utils.cc @@ -0,0 +1,120 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/eager/utils.h" +#include "paddle/fluid/eager/api/all.h" + +namespace egr { +/* ---- Tensor -> Var ---- */ +std::vector> EagerUtils::SyncToVars( + const egr::EagerTensor& tensor) { + // TODO(jiabin): No const cast here. We should call SyncToVar in Python_C + // wrapper + const_cast(&tensor)->SyncToVar( + paddle::framework::proto::VarType_Type_LOD_TENSOR); + return {std::make_shared(tensor)}; +} + +std::vector> EagerUtils::SyncToVars( + const std::vector& tensors) { + // TODO(jiabin): No const cast here. We should call SyncToVar in Python_C + // wrapper + std::vector> res; + size_t num = tensors.size(); + res.reserve(num); + for (size_t i = 0; i < num; i++) { + const_cast(&(tensors[i])) + ->SyncToVar(paddle::framework::proto::VarType_Type_LOD_TENSOR); + res.emplace_back(new EagerTensor(tensors[i])); + } + return res; +} + +/* ---- VarBase -> Tensor ---- */ +std::vector> EagerUtils::SyncToTensors( + const egr::EagerTensor& tensor) { + // TODO(jiabin): No const cast here. We should call SyncToTensor in Python_C + // wrapper + const_cast(&tensor)->SyncToTensor(); + return {std::make_shared(tensor)}; +} + +std::vector> EagerUtils::SyncToTensors( + const std::vector& tensors) { + // TODO(jiabin): No const cast here. We should call SyncToTensor in Python_C + // wrapper + std::vector> res; + size_t num = tensors.size(); + res.reserve(num); + for (size_t i = 0; i < num; i++) { + const_cast(&(tensors[i]))->SyncToTensor(); + res.emplace_back(new EagerTensor(tensors[i])); + } + return res; +} + +std::vector> EagerUtils::ConstructDuplicableOutput( + const size_t num) { + std::vector> res; + res.reserve(num); + for (size_t i = 0; i < num; i++) { + res.emplace_back( + new EagerTensor(egr::Controller::Instance().GenerateUniqueName())); + } + return res; +} + +std::vector EagerUtils::GetOutputs( + const std::vector>& outs) { + std::vector res; + res.reserve(outs.size()); + for (const auto& out : outs) { + PADDLE_ENFORCE_NOT_NULL( + out.get(), paddle::platform::errors::Fatal( + "Eager Tensor %s is null and cannot be copied. " + "We are tring to Get Output tensor from its " + "shared_ptr, this error may indicate some outputs " + "are nullptr", + out->name())); + res.emplace_back((*(out.get()))); + } + return res; +} + +egr::EagerTensor EagerUtils::GetOutput( + const std::shared_ptr& out) { + PADDLE_ENFORCE_NOT_NULL( + out.get(), paddle::platform::errors::Fatal( + "Eager Tensor %s is null and cannot be copied. We " + "are tring to Get Output tensor from its shared_ptr, " + "this error may indicate output is nullptr", + out->name())); + return EagerTensor((*(out.get()))); +} + +AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) { + auto* p_autograd_meta = target.get_autograd_meta(); + PADDLE_ENFORCE(p_autograd_meta, + paddle::platform::errors::Fatal( + "Null autograd_meta gotten from unsafe_autograd_meta(), " + "if you are using unsafe_autograd_meta, please make sure " + "your tensor's autograd_meta is set")); + return static_cast(p_autograd_meta); +} + +std::pair EagerUtils::OutRankInfo( + const egr::EagerTensor& target) { + return unsafe_autograd_meta(target)->OutRankInfo(); +} +} // namespace egr diff --git a/paddle/fluid/eager/utils.h b/paddle/fluid/eager/utils.h new file mode 100644 index 00000000000..4e8461e6600 --- /dev/null +++ b/paddle/fluid/eager/utils.h @@ -0,0 +1,126 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/grad_node_info.h" + +#include "paddle/pten/api/all.h" + +namespace egr { + +/** + * EagerUtils is utils used to do some static conversion or autograd + * members access, this class is desinged to be a full static functional + * utils class + * **/ + +template +class IterHelper { + virtual void visit(ElementType element) = 0; + + void visit(std::vector* elements) { + for (auto element : *elements) visit(element); + } + + template + void apply() {} + + public: + template + void apply(T&& arg, Args&&... args) { + visit(std::forward(arg)); + return apply(std::forward(args)...); + } + virtual ~IterHelper() = default; +}; + +class ComputeRequireGradIter : public IterHelper { + public: + bool RequireGrad() { return require_grad_; } + + private: + void visit(AutogradMeta* element) override { + bool stop_gradient = element->StopGradient(); + if (!stop_gradient) require_grad_ = true; + } + + bool require_grad_ = false; +}; + +class PassStopGradientIter : public IterHelper { + public: + void SetStopGradient(bool stop_gradient) { stop_gradient_ = stop_gradient; } + + private: + void visit(AutogradMeta* element) override { + if (!element) { + // TODO(jiabin): Add Tensor name here when we supported. + VLOG(2) << "Tensor is NULL"; + return; + } + element->SetStopGradient(stop_gradient_); + } + + bool stop_gradient_ = true; +}; + +class EagerUtils { + public: + /** + * We have to use autograd_meta and multi_autograd_meta to initialize + * autograd_meta for tensor, since we can't init it in + * egr::EagerTensor's + * constructor (it's abstract class there) + * + * **/ + template + static bool ComputeRequireGrad(T trace_backward, Args&&... args) { + if (!trace_backward) return false; + + auto iter = ComputeRequireGradIter(); + iter.apply(std::forward(args)...); + + return iter.RequireGrad(); + } + + template + static void PassStopGradient(T stop_gradient, Args&&... args) { + auto iter = PassStopGradientIter(); + iter.SetStopGradient(stop_gradient); + iter.apply(std::forward(args)...); + } + static std::pair OutRankInfo(const egr::EagerTensor& target); + // This method will return an AutogradMeta pointer unsafely. + static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target); + + // Intermidate needed remove this once we don't need legacy + static std::vector> SyncToVars( + const egr::EagerTensor& tensor); + static std::vector> SyncToVars( + const std::vector& tensors); + static std::vector> SyncToTensors( + const egr::EagerTensor& tensor); + static std::vector> SyncToTensors( + const std::vector& tensors); + static std::vector> ConstructDuplicableOutput( + const size_t num); + static std::vector GetOutputs( + const std::vector>& outs); + static egr::EagerTensor GetOutput(const std::shared_ptr& outs); +}; + +} // namespace egr -- GitLab