eager_amp_auto_cast.h 5.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"

namespace egr {

static inline bool NeedCast(const paddle::experimental::Tensor& tensor,
                            const paddle::experimental::DataType& dst_dtype) {
C
Chen Weihang 已提交
23
  auto place = tensor.place();
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
  auto data_type = tensor.dtype();
  if (paddle::platform::is_gpu_place(place) ||
      paddle::platform::is_cuda_pinned_place(place) ||
      paddle::platform::is_xpu_place(place) ||
      paddle::platform::is_mlu_place(place) ||
      paddle::platform::is_npu_place(place) ||
      paddle::platform::is_npu_pinned_place(place)) {
    // CudaPinndePlace is added for varbase created by dataloader
    if ((data_type == paddle::experimental::DataType::FLOAT32 ||
         data_type == paddle::experimental::DataType::FLOAT16 ||
         data_type == paddle::experimental::DataType::BFLOAT16) &&
        (data_type != dst_dtype)) {
      return true;
    }
  }
  return false;
}

Z
zhangkaihuo 已提交
42 43 44 45 46 47
inline paddle::experimental::Tensor Cast(
    const paddle::experimental::Tensor& input,
    const paddle::experimental::DataType& dst_dtype,
    const bool trace_backward = true) {
  if (input.is_sparse_coo_tensor() || input.is_sparse_csr_tensor()) {
    if (trace_backward) {
48
      return sparse::cast_dygraph_function(
Z
zhangkaihuo 已提交
49 50 51 52 53 54 55
          input, paddle::experimental::DataType::UNDEFINED, dst_dtype);
    } else {
      return paddle::experimental::sparse::cast(
          input, paddle::experimental::DataType::UNDEFINED, dst_dtype);
    }
  } else {
    if (trace_backward) {
56
      return cast_dygraph_function(input, dst_dtype);
Z
zhangkaihuo 已提交
57 58 59 60 61 62
    } else {
      return paddle::experimental::cast(input, dst_dtype);
    }
  }
}

63 64 65
inline std::vector<paddle::experimental::Tensor> EagerAmpAutoCasts(
    const std::string& inputs_name,
    const std::vector<paddle::experimental::Tensor>& inputs,
66
    const paddle::experimental::DataType& dst_dtype,
67 68
    std::string op_name,
    bool trace_backward = true) {
69 70 71 72 73 74
  VLOG(6) << "AMP AmpAutoCasts:"
          << " inputs(" << inputs_name << ") dst_dtype("
          << paddle::framework::DataType2String(dst_dtype) << ").";
  std::vector<paddle::experimental::Tensor> inputs_casted;
  for (auto& input : inputs) {
    if (NeedCast(input, dst_dtype)) {
Z
zhangkaihuo 已提交
75
      inputs_casted.emplace_back(std::move(Cast(input, dst_dtype)));
76 77 78 79 80 81 82 83
    } else {
      inputs_casted.emplace_back(input);
    }
  }
  return inputs_casted;
}

inline paddle::experimental::Tensor EagerAmpAutoCast(
84 85
    const std::string& input_name,
    const paddle::experimental::Tensor& input,
86
    const paddle::experimental::DataType& dst_dtype,
87 88
    const std::string& op_name,
    bool trace_backward = true) {
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
  VLOG(6) << "AMP AmpAutoCasts:"
          << " input(" << input_name << ") dst_dtype("
          << paddle::framework::DataType2String(dst_dtype) << ").";
  if (dst_dtype == paddle::experimental::DataType::FLOAT16) {
    if (op_name == "run_program") {
      return input;
    }
    if ((op_name == "batch_norm" || op_name == "layer_norm" ||
         op_name == "sync_batch_norm") &&
        input_name != "x") {
      return input;
    }
    if ((op_name == "fused_attention" || op_name == "fused_feedforward")) {
      if (input_name == "LnScale" || input_name == "LnBias" ||
          input_name == "Ln2Scale" || input_name == "Ln2Bias" ||
          input_name == "Ln1Scale" || input_name == "Ln1Bias") {
        return input;
      }
    }
  }
  if (NeedCast(input, dst_dtype)) {
Z
zhangkaihuo 已提交
110
    return Cast(input, dst_dtype, trace_backward);
111 112 113 114
  }
  return input;
}

115 116 117 118
inline paddle::optional<paddle::experimental::Tensor> EagerAmpAutoCast(
    const std::string& input_name,
    const paddle::optional<paddle::experimental::Tensor>& input,
    const paddle::experimental::DataType& dst_dtype,
119 120
    const std::string& op_name,
    bool trace_backward = true) {
121
  if (input) {
122 123
    return EagerAmpAutoCast(
        input_name, *input, dst_dtype, op_name, trace_backward);
124 125 126 127
  }
  return paddle::none;
}

128 129 130 131 132 133 134 135 136 137 138 139 140 141
inline paddle::optional<std::vector<paddle::experimental::Tensor>>
EagerAmpAutoCasts(
    const std::string& inputs_name,
    const paddle::optional<std::vector<paddle::experimental::Tensor>>& inputs,
    const paddle::experimental::DataType& dst_dtype,
    std::string op_name,
    bool trace_backward = true) {
  if (inputs) {
    return EagerAmpAutoCasts(
        inputs_name, *inputs, dst_dtype, op_name, trace_backward);
  }
  return paddle::optional<std::vector<paddle::experimental::Tensor>>();
}

142
}  // namespace egr