utils.h 9.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

17
#include "paddle/fluid/eager/api/utils/tensor_utils.h"
18 19 20 21
#include "paddle/fluid/eager/autograd_meta.h"
#include "paddle/fluid/eager/eager_tensor.h"
#include "paddle/fluid/eager/grad_node_info.h"

22
#include "paddle/phi/api/all.h"
23 24 25

namespace egr {

26 27
class TensorWrapper;

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
/**
 * EagerUtils is utils used to do some static conversion or autograd
 * members access, this class is desinged to be a full static functional
 * utils class
 * **/

template <typename ElementType>
class IterHelper {
  virtual void visit(ElementType element) = 0;

  void visit(std::vector<ElementType>* elements) {
    for (auto element : *elements) visit(element);
  }

  template <typename... Args>
  void apply() {}

 public:
  template <typename T, typename... Args>
  void apply(T&& arg, Args&&... args) {
    visit(std::forward<T>(arg));
    return apply(std::forward<Args>(args)...);
  }
  virtual ~IterHelper() = default;
};

class ComputeRequireGradIter : public IterHelper<AutogradMeta*> {
 public:
  bool RequireGrad() { return require_grad_; }

 private:
  void visit(AutogradMeta* element) override {
60 61 62
    // Dispensable Tensors feeds in nullptr autograd_meta
    if (!element) return;

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
    bool stop_gradient = element->StopGradient();
    if (!stop_gradient) require_grad_ = true;
  }

  bool require_grad_ = false;
};

class PassStopGradientIter : public IterHelper<AutogradMeta*> {
 public:
  void SetStopGradient(bool stop_gradient) { stop_gradient_ = stop_gradient; }

 private:
  void visit(AutogradMeta* element) override {
    if (!element) {
      // TODO(jiabin): Add Tensor name here when we supported.
      VLOG(2) << "Tensor is NULL";
      return;
    }
81
    element->WeakSetStopGradient(stop_gradient_);
82 83 84 85 86 87 88 89 90 91
  }

  bool stop_gradient_ = true;
};

class EagerUtils {
 public:
  /**
   * We have to use autograd_meta and multi_autograd_meta to initialize
   * autograd_meta for tensor, since we can't init it in
92
   * egr::EagerVariable's
93 94 95
   * constructor (it's abstract class there)
   *
   * **/
96
  static AutogradMeta* autograd_meta(paddle::experimental::Tensor* target);
97

98
  static std::vector<AutogradMeta*> autograd_meta(
99
      std::vector<paddle::experimental::Tensor>* targets);
100

W
wanghuancoder 已提交
101 102 103
  static std::vector<AutogradMeta*> autograd_meta(
      std::vector<paddle::experimental::Tensor*>* targets);

104 105
  static std::pair<size_t, size_t> OutRankInfo(
      const paddle::experimental::Tensor& target);
106 107

  static std::shared_ptr<GradNodeBase> grad_node(
108
      const paddle::experimental::Tensor& target);
109 110
  static paddle::experimental::Tensor* mutable_grad(
      const paddle::experimental::Tensor& target);
111 112 113 114 115 116 117 118 119 120 121 122 123 124

  // Set history is used to set backward info during forward process, it will
  // set forward var's autograd meta's grad node as current backward node.
  static void SetHistory(std::vector<AutogradMeta*>* autograd_metas,
                         const std::shared_ptr<GradNodeBase>& grad_node);
  static void SetHistory(AutogradMeta* autograd_meta,
                         const std::shared_ptr<GradNodeBase>& grad_node);

  // This is used for Set vector of tensors' rank
  static void SetOutRankWithSlot(std::vector<AutogradMeta*>* targets,
                                 size_t slot_id);
  static void SetOutRankWithSlot(AutogradMeta* target, size_t slot_id);

  // This method will return an AutogradMeta pointer unsafely.
125 126
  static AutogradMeta* nullable_autograd_meta(
      const paddle::experimental::Tensor& target);
H
hong 已提交
127
  static AutogradMeta* nullable_autograd_meta(
128
      const paddle::optional<paddle::experimental::Tensor>& target);
129
  static std::vector<AutogradMeta*> nullable_autograd_meta(
130
      const std::vector<paddle::experimental::Tensor>& targets);
W
wanghuancoder 已提交
131 132
  static std::vector<AutogradMeta*> nullable_autograd_meta(
      const std::vector<paddle::experimental::Tensor*>& targets);
133 134
  static AutogradMeta* unsafe_autograd_meta(
      const paddle::experimental::Tensor& target);
135
  static std::vector<AutogradMeta*> unsafe_autograd_meta(
136
      const std::vector<paddle::experimental::Tensor>& targets);
137

138 139
  template <typename T, typename... Args>
  static bool ComputeRequireGrad(T trace_backward, Args&&... args) {
W
wanghuancoder 已提交
140 141 142 143
    if (!trace_backward) {
      VLOG(6) << "Do not require grad because trace_backward = false";
      return false;
    }
144 145 146 147 148 149 150 151 152 153 154 155 156 157

    auto iter = ComputeRequireGradIter();
    iter.apply(std::forward<Args>(args)...);

    return iter.RequireGrad();
  }

  template <typename T, typename... Args>
  static void PassStopGradient(T stop_gradient, Args&&... args) {
    auto iter = PassStopGradientIter();
    iter.SetStopGradient(stop_gradient);
    iter.apply(std::forward<Args>(args)...);
  }

158 159 160 161 162 163 164 165 166 167 168 169 170
  static void CheckInplace(const paddle::experimental::Tensor& target,
                           const AutogradMeta* autograd_meta,
                           bool require_any_grad) {
    if (require_any_grad && autograd_meta) {
      PADDLE_ENFORCE_EQ(!autograd_meta->StopGradient() &&
                            egr::egr_utils_api::IsLeafTensor(target),
                        false, paddle::platform::errors::InvalidArgument(
                                   "Leaf Var (%s) that doesn't stop gradient "
                                   "can't use inplace strategy.",
                                   target.name()));
    }
  }

171 172 173 174
  // View Strategy
  static void HandleViewBetweenInputAndOutput(
      const std::shared_ptr<EagerVariable>& input_var,
      const std::shared_ptr<EagerVariable>& view_output_var);
175 176 177
  static void HandleViewBetweenInputAndOutput(
      const paddle::experimental::Tensor& input_tensor,
      paddle::experimental::Tensor* view_output_tensor);
178

179
  // TensorWrapper Utils
180
  static paddle::experimental::Tensor RecoverTensorWrapper(TensorWrapper* tw);
181
  static std::vector<paddle::experimental::Tensor> RecoverTensorWrapper(
182
      std::vector<TensorWrapper>* tw);
183

184
  // Intermidate needed remove this once we don't need legacy
185
  // Inner Method
186
  static std::shared_ptr<egr::EagerVariable> TrySyncToVar(
187 188
      const paddle::experimental::Tensor& tensor);
  // Basic Input
189
  static std::vector<std::shared_ptr<egr::EagerVariable>> TrySyncToVars(
190 191
      const paddle::experimental::Tensor& tensor);
  // Basic Output
192
  static std::vector<std::shared_ptr<egr::EagerVariable>> TrySyncToVars(
193 194
      paddle::experimental::Tensor* tensor);
  // Multi Output
195
  static std::vector<std::shared_ptr<egr::EagerVariable>> TrySyncToVars(
196 197
      const std::vector<paddle::experimental::Tensor*>& tensors);
  // Multi Input
198
  static std::vector<std::shared_ptr<egr::EagerVariable>> TrySyncToVars(
199 200
      const std::vector<paddle::experimental::Tensor>& tensors);
  // Construct empty output
201 202
  static std::vector<std::shared_ptr<EagerVariable>> CreateVars(
      const size_t num);
203 204
  // Construct Tensor From var
  static std::vector<paddle::experimental::Tensor> GetOutputs(
205
      const std::vector<std::shared_ptr<EagerVariable>>& outs);
206
  static paddle::experimental::Tensor GetOutput(
207
      const std::shared_ptr<EagerVariable>& out);
208 209 210
  static void GetOutput(const std::shared_ptr<EagerVariable>& out,
                        paddle::experimental::Tensor* out_var);
  static void GetOutputs(
211
      const std::vector<std::shared_ptr<EagerVariable>>& outs,
212 213 214 215 216 217 218 219 220 221 222 223 224 225
      std::vector<paddle::experimental::Tensor>* result);
  static void GetOutputs(
      const std::vector<std::shared_ptr<EagerVariable>>& outs,
      const std::vector<paddle::experimental::Tensor*>& out_var);
  static void GetOutputs(const std::shared_ptr<EagerVariable>& out,
                         std::vector<paddle::experimental::Tensor>* result);
  static void GetOutputs(
      const std::shared_ptr<EagerVariable>& out,
      const std::vector<paddle::experimental::Tensor*>& out_var);

  static void Output2Result(
      const std::vector<paddle::experimental::Tensor*>& out_var,
      std::vector<paddle::experimental::Tensor>* result);

226 227 228 229
  // end Intermidate needed

  static void CheckAndRetainGrad(const paddle::experimental::Tensor& tensor);
  static void CheckAndRetainGrad(
230
      const std::vector<paddle::experimental::Tensor>& tensors);
W
wanghuancoder 已提交
231 232
  static void CheckAndRetainGrad(
      const std::vector<paddle::experimental::Tensor*>& tensors);
233 234
  static std::shared_ptr<egr::GradNodeBase> GetGradAccumulationNode(
      const paddle::experimental::Tensor& tensor);
235 236 237 238 239

  /**
    * Fill Zero
    * **/
  static void FillZeroForEmptyGradInputs(
240 241 242 243
      paddle::small_vector<std::vector<paddle::experimental::Tensor>,
                           kSlotSmallVectorSize>* out_grads,
      const paddle::small_vector<std::vector<GradSlotMeta>,
                                 kSlotSmallVectorSize>& grad_out_metas);
244 245 246 247 248 249 250
  static void FillZeroForEmptyGradInput(paddle::experimental::Tensor* in_grad,
                                        const GradSlotMeta& grad_in_meta);
  static void FillZeroForEmptyOptionalGradInput(
      paddle::experimental::Tensor* in_grad, const GradSlotMeta& grad_in_meta);
  static void FillZeroForEmptyGradInput(
      std::vector<paddle::experimental::Tensor>* in_grads,
      const std::vector<GradSlotMeta>& grad_in_metas);
251 252 253
};

}  // namespace egr