hook_test.cc 7.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <sstream>

#include "glog/logging.h"
#include "gtest/gtest.h"

#include "paddle/fluid/eager/accumulation/accumulation_node.h"
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
#include "paddle/fluid/eager/autograd_meta.h"
#include "paddle/fluid/eager/backward.h"
#include "paddle/fluid/eager/grad_node_info.h"

#include "paddle/fluid/eager/api/all.h"

28 29
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/tensor_meta.h"
30

31
#include "paddle/fluid/eager/hooks.h"
32 33
#include "paddle/fluid/eager/tests/test_utils.h"

34 35 36 37
#include "paddle/phi/core/kernel_registry.h"

PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);

38
namespace egr {
39

40 41
paddle::experimental::Tensor hook_function(
    const paddle::experimental::Tensor& t) {
42
  auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
43

44 45
  auto ret_meta = phi::DenseTensorMeta(t_dense->dtype(), t_dense->dims(),
                                       t_dense->layout());
46
  auto place = t_dense->place();
47 48
  size_t bytes_size = phi::product(t_dense->dims()) * SizeOf(t_dense->dtype());
  auto ret_dense = std::make_shared<phi::DenseTensor>(
Z
zyfncg 已提交
49
      paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
50

51 52
  float* t_ptr = t_dense->mutable_data<float>(place);
  float* ret_ptr = ret_dense->mutable_data<float>(place);
53 54 55 56
  for (int i = 0; i < ret_dense->numel(); i++) {
    ret_ptr[i] = t_ptr[i] + 3.0;
  }

57
  auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
58
  paddle::experimental::Tensor ret = paddle::experimental::Tensor();
59 60 61 62 63 64
  ret.set_impl(ret_impl);

  return ret;
}

TEST(RetainGrad, HookBeforeRetainGrad) {
65
  eager_test::InitEnv(paddle::platform::CPUPlace());
66 67

  // Prepare Inputs
68
  std::vector<paddle::experimental::Tensor> target_tensors;
69
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
70 71

  // Create Target Tensor
72
  paddle::experimental::Tensor tensor = egr_utils_api::CreateTensorWithValue(
73 74
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
75
  target_tensors.emplace_back(std::move(tensor));
76
  paddle::experimental::Tensor& target_tensor = target_tensors[0];
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93

  // Create ScaleNode
  auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
  scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);

  // Set grad in/out meta for node0
  scale_node_ptr->SetDefaultGradInOutMeta();

  // Connect Input Tensor and ScaleNode via AutoGradMeta
  // Apply RetainGrad
  {
    // ScaleNode Hook: +3

    auto auto_grad_meta = std::make_shared<AutogradMeta>();
    auto_grad_meta->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
94
    auto_grad_meta->SetStopGradient(false);
95 96 97 98
    target_tensor.set_autograd_meta(
        std::dynamic_pointer_cast<paddle::experimental::AbstractAutogradMeta>(
            auto_grad_meta));

99 100
    egr_utils_api::RegisterGradientHookForTensor(
        target_tensor, std::make_shared<egr::CppTensorHook>(hook_function));
101 102
    egr_utils_api::RetainGradForTensor(
        target_tensor);  // result: 1.0 + 3.0 = 4.0
103 104
    egr_utils_api::RetainGradForTensor(
        target_tensor);  // result: 1.0 + 3.0 = 4.0
105 106 107
  }

  // Retain Grad for leaf tensor1
108
  paddle::experimental::Tensor leaf_tensor = paddle::experimental::Tensor();
109 110
  {
    // AccumulationNode Hook: +3
111 112
    auto tmp_tensor0 = paddle::experimental::Tensor();
    auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
113

114
    auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(auto_grad_meta);
115 116 117

    auto_grad_meta->SetStopGradient(false);
    auto_grad_meta->SetGradNode(acc_node_ptr);
118
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
119 120
    std::vector<egr::AutogradMeta*> res = {auto_grad_meta};
    scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
121

122 123
    leaf_tensor.set_autograd_meta(
        std::dynamic_pointer_cast<paddle::experimental::AbstractAutogradMeta>(
124
            tmp_tensor0.mutable_autograd_meta()));
125

126 127
    egr_utils_api::RegisterGradientHookForTensor(
        leaf_tensor, std::make_shared<egr::CppTensorHook>(hook_function));
128 129
    egr_utils_api::RetainGradForTensor(
        leaf_tensor);  // result: 4.0*5.0 + 3.0 = 23.0
130 131
  }

132
  Backward(target_tensors, {});
133

134 135
  eager_test::CompareGradTensorWithValue<float>(target_tensor, 4.0);
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
136 137 138
}

TEST(RetainGrad, HookAfterRetainGrad) {
139
  eager_test::InitEnv(paddle::platform::CPUPlace());
140 141

  // Prepare Inputs
142
  std::vector<paddle::experimental::Tensor> target_tensors;
143
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
144 145

  // Create Target Tensor
146
  paddle::experimental::Tensor tensor = egr_utils_api::CreateTensorWithValue(
147 148
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
149
  target_tensors.emplace_back(std::move(tensor));
150
  paddle::experimental::Tensor& target_tensor = target_tensors[0];
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166

  // Create ScaleNode
  auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
  scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
  // Set grad in/out meta for node0
  scale_node_ptr->SetDefaultGradInOutMeta();

  // Connect Input Tensor and ScaleNode via AutoGradMeta
  // Apply RetainGrad
  {
    // ScaleNode Hook: +3

    auto auto_grad_meta = std::make_shared<AutogradMeta>();
    auto_grad_meta->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
167
    auto_grad_meta->SetStopGradient(false);
168 169 170 171
    target_tensor.set_autograd_meta(
        std::dynamic_pointer_cast<paddle::experimental::AbstractAutogradMeta>(
            auto_grad_meta));

172
    egr_utils_api::RetainGradForTensor(target_tensor);  // result: 1.0
173 174
    egr_utils_api::RegisterGradientHookForTensor(
        target_tensor, std::make_shared<egr::CppTensorHook>(hook_function));
175 176 177
  }

  // Retain Grad for leaf tensor1
178
  paddle::experimental::Tensor leaf_tensor = paddle::experimental::Tensor();
179 180
  {
    // AccumulationNode Hook: +3
181 182 183
    auto tmp_tensor0 = paddle::experimental::Tensor();
    auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
    auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(auto_grad_meta);
184 185
    auto_grad_meta->SetGradNode(acc_node_ptr);
    auto_grad_meta->SetStopGradient(false);
186
    scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
187

188 189 190
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
    leaf_tensor.set_autograd_meta(
        std::dynamic_pointer_cast<paddle::experimental::AbstractAutogradMeta>(
191
            tmp_tensor0.mutable_autograd_meta()));
192

193 194
    egr_utils_api::RegisterGradientHookForTensor(
        leaf_tensor, std::make_shared<egr::CppTensorHook>(hook_function));
195 196
  }

197
  Backward(target_tensors, {});
198 199
  eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
200
}
201
}  // namespace egr