backward_test.cc 11.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <sstream>

#include "glog/logging.h"
#include "gtest/gtest.h"

#include "paddle/fluid/eager/accumulation/accumulation_node.h"
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
#include "paddle/fluid/eager/api/utils/tensor_utils.h"
#include "paddle/fluid/eager/autograd_meta.h"
#include "paddle/fluid/eager/backward.h"
#include "paddle/fluid/eager/grad_node_info.h"
#include "paddle/fluid/eager/tests/test_utils.h"

#include "paddle/fluid/eager/api/all.h"

30 31
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/tensor_meta.h"
32

33 34 35
#include "paddle/phi/core/kernel_registry.h"

PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
36
PD_DECLARE_KERNEL(copy, CPU, ALL_LAYOUT);
37

38
namespace egr {
39 40 41

TEST(Backward, SingleNodeEmptyGrad) {
  // Prepare Device Contexts
42
  eager_test::InitEnv(paddle::platform::CPUPlace());
43 44

  // Prepare Inputs
45
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
46 47

  // Create Target Tensor
48 49
  paddle::experimental::Tensor target_tensor =
      egr_utils_api::CreateTensorWithValue(
50 51
          ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
          phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
52

53
  paddle::experimental::Tensor leaf_tensor;
54 55 56 57 58 59 60 61 62 63 64
  {
    // Create Scale Node
    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
    node0_ptr->SetAttributes_scale(5.0 /*scale*/);

    // Set grad in/out meta
    node0_ptr->SetDefaultGradInOutMeta();
    AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&target_tensor);
    auto_grad_meta->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
65
    auto_grad_meta->SetStopGradient(false);
66

67 68
    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);

69
    // Connect Tensor and AccumulationNode via AutoGradMeta
70 71
    auto acc_node_ptr =
        std::make_shared<egr::GradNodeAccumulation>(auto_grad_meta1);
72 73 74 75

    auto_grad_meta1->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
76
    auto_grad_meta1->SetStopGradient(false);
77

78
    std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
79
    node0_ptr->AddEdges(&res, 0);
80
  }
81
  std::vector<paddle::experimental::Tensor> outs = {target_tensor};
82
  // Run Backward
83
  Backward(outs, {});
84 85

  // Check Output Value
86
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 5.0);
87 88 89 90
}

TEST(Backward, SingleNodeCustomGrad) {
  // Prepare Device Contexts
91
  eager_test::InitEnv(paddle::platform::CPUPlace());
92 93

  // Prepare Inputs
94
  std::vector<paddle::experimental::Tensor> target_tensors;
95
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
96 97

  // Create Target Tensor
98
  paddle::experimental::Tensor tensor = egr_utils_api::CreateTensorWithValue(
99 100
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
101 102
  target_tensors.emplace_back(std::move(tensor));

103
  std::vector<paddle::experimental::Tensor> grad_tensors;
104
  // Create Grad Tensor
105 106
  paddle::experimental::Tensor grad_tensor =
      egr_utils_api::CreateTensorWithValue(
107 108
          ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
          phi::DataLayout::NCHW, 10.0 /*value*/, false /*is_leaf*/);
109 110
  grad_tensors.emplace_back(std::move(grad_tensor));

111
  paddle::experimental::Tensor leaf_tensor;
112 113 114 115 116 117 118 119 120 121 122 123 124 125
  {
    // Create Scale Node
    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
    node0_ptr->SetAttributes_scale(5.0 /*scale*/);

    // Set grad in/out meta
    node0_ptr->SetDefaultGradInOutMeta();

    // Connect Tensor and Node via AutoGradMeta
    AutogradMeta* auto_grad_meta =
        EagerUtils::autograd_meta(&(target_tensors[0]));
    auto_grad_meta->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
126
    auto_grad_meta->SetStopGradient(false);
127 128

    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
129 130 131 132
    // Connect Tensor and AccumulationNode via AutoGradMeta
    auto acc_node_ptr =
        std::make_shared<egr::GradNodeAccumulation>(auto_grad_meta1);

133 134 135
    auto_grad_meta1->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
136 137
    auto_grad_meta1->SetStopGradient(false);
    std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
138
    node0_ptr->AddEdges(&res, 0);
139 140 141
  }

  // Run Backward
142
  Backward(target_tensors, grad_tensors);
143 144

  // Check Output Value
145
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
146 147 148 149 150 151 152 153 154 155 156
}

/*
Node1
  |
Node0
  |
 inp0
*/
TEST(Backward, LinearNodes) {
  // Prepare Device Contexts
157
  eager_test::InitEnv(paddle::platform::CPUPlace());
158 159

  // Prepare Inputs
160
  std::vector<paddle::experimental::Tensor> target_tensors;
161
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
162 163

  // Create Target Tensor
164
  paddle::experimental::Tensor tensor = egr_utils_api::CreateTensorWithValue(
165 166
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
167 168
  target_tensors.emplace_back(std::move(tensor));

169
  paddle::experimental::Tensor leaf_tensor;
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
  {
    // Create Node0
    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
    node0_ptr->SetAttributes_scale(5.0 /*scale*/);

    // Set grad in/out meta for node0
    node0_ptr->SetDefaultGradInOutMeta();

    // Create Node1
    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
    node1_ptr->SetAttributes_scale(10.0 /*scale*/);

    // Set grad in/out meta for node1
    node1_ptr->SetDefaultGradInOutMeta();

    // Connect Input Tensor and Node0 via AutoGradMeta
    AutogradMeta* auto_grad_meta =
        EagerUtils::autograd_meta(&(target_tensors[0]));
    auto_grad_meta->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
191
    auto_grad_meta->SetStopGradient(false);
192 193
    // Connect Node0 -> Node1 via Edge
    auto meta0 = egr::AutogradMeta();
194
    meta0.SetStopGradient(false);
195 196
    meta0.SetSingleOutRankWithSlot(0, 0);
    meta0.SetGradNode(node1_ptr);
197 198
    std::vector<egr::AutogradMeta*> res0 = {&meta0};
    node0_ptr->AddEdges(&res0, 0);
199

200
    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
201
    // Connect Tensor and AccumulationNode via AutoGradMeta
202 203
    auto acc_node_ptr =
        std::make_shared<egr::GradNodeAccumulation>(auto_grad_meta1);
204 205 206 207 208

    auto_grad_meta1->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);

209 210
    auto_grad_meta1->SetStopGradient(false);
    std::vector<egr::AutogradMeta*> res1 = {auto_grad_meta1};
211
    node1_ptr->AddEdges(&res1, 0);
212 213 214
  }

  // Use Empty Grad Tensor
215
  Backward(target_tensors, {});
216 217

  // Check Output Value
218
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
219 220 221 222 223 224 225 226 227 228 229
}

/*
    Node2
    |   |
Node0   Node1
  |      |
 inp0   inp1
*/
TEST(Backward, WithAccumulation) {
  // Prepare Device Contexts
230
  eager_test::InitEnv(paddle::platform::CPUPlace());
231 232

  // Prepare Inputs
233
  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
234 235

  // Create Target Tensor
236 237
  std::vector<paddle::experimental::Tensor> target_tensors;
  paddle::experimental::Tensor tensor0 = egr_utils_api::CreateTensorWithValue(
238 239
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
240
  paddle::experimental::Tensor tensor1 = egr_utils_api::CreateTensorWithValue(
241 242
      ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
      phi::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
243 244 245 246
  target_tensors.emplace_back(std::move(tensor0));
  target_tensors.emplace_back(std::move(tensor1));

  // Create Grad Tensor
247 248 249
  std::vector<paddle::experimental::Tensor> grad_tensors;
  paddle::experimental::Tensor grad_tensor0 =
      egr_utils_api::CreateTensorWithValue(
250 251
          ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
          phi::DataLayout::NCHW, 5.0 /*value*/, false /*is_leaf*/);
252 253
  paddle::experimental::Tensor grad_tensor1 =
      egr_utils_api::CreateTensorWithValue(
254 255
          ddim, paddle::platform::CPUPlace(), phi::DataType::FLOAT32,
          phi::DataLayout::NCHW, 10.0 /*value*/, false /*is_leaf*/);
256 257 258
  grad_tensors.emplace_back(std::move(grad_tensor0));
  grad_tensors.emplace_back(std::move(grad_tensor1));

259
  paddle::experimental::Tensor leaf_tensor;
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
  {
    // Create Node0
    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
    node0_ptr->SetDefaultGradInOutMeta();

    // Create Node1
    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
    node1_ptr->SetAttributes_scale(10.0 /*scale*/);
    node1_ptr->SetDefaultGradInOutMeta();
    // Create Node2
    auto node2_ptr = std::make_shared<GradNodeScale>(1, 1);
    node2_ptr->SetAttributes_scale(20.0 /*scale*/);
    node2_ptr->SetDefaultGradInOutMeta();
    // Connect Inp0 and Node0 via AutoGradMeta
    AutogradMeta* auto_grad_meta0 =
        EagerUtils::autograd_meta(&(target_tensors[0]));
    auto_grad_meta0->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
    auto_grad_meta0->SetSingleOutRankWithSlot(0, 0);
280
    auto_grad_meta0->SetStopGradient(false);
281 282 283 284 285 286
    // Connect Inp1 and Node1 via AutoGradMeta
    AutogradMeta* auto_grad_meta1 =
        EagerUtils::autograd_meta(&(target_tensors[1]));
    auto_grad_meta1->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(node1_ptr));
    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
287
    auto_grad_meta1->SetStopGradient(false);
288 289 290

    // Connect Node0 -> Node2 via Edge
    auto meta0 = egr::AutogradMeta();
291
    meta0.SetStopGradient(false);
292 293
    meta0.SetSingleOutRankWithSlot(0, 0);
    meta0.SetGradNode(node2_ptr);
294 295
    std::vector<egr::AutogradMeta*> res0 = {&meta0};
    node0_ptr->AddEdges(&res0, 0);
296 297 298

    // Connect Node1 -> Node2 via Edge
    auto meta1 = egr::AutogradMeta();
299
    meta1.SetStopGradient(false);
300 301
    meta1.SetSingleOutRankWithSlot(0, 0);
    meta1.SetGradNode(node2_ptr);
302 303
    std::vector<egr::AutogradMeta*> res1 = {&meta1};
    node1_ptr->AddEdges(&res1, 0);
304

305
    AutogradMeta* auto_grad_meta2 = EagerUtils::autograd_meta(&leaf_tensor);
306
    // Connect Tensor and AccumulationNode via AutoGradMeta
307 308
    auto acc_node_ptr =
        std::make_shared<egr::GradNodeAccumulation>(auto_grad_meta2);
309 310 311 312 313

    auto_grad_meta2->SetGradNode(
        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
    auto_grad_meta2->SetSingleOutRankWithSlot(0, 0);

314 315
    auto_grad_meta2->SetStopGradient(false);
    std::vector<egr::AutogradMeta*> res2 = {auto_grad_meta2};
316
    node2_ptr->AddEdges(&res2, 0);
317 318
  }

319
  Backward(target_tensors, grad_tensors);
320

321
  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 2500.0);
322 323
}

324
}  // namespace egr