cinn_launch_op.h 9.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#include <memory>
#include <string>
#include <unordered_map>
20
#include <unordered_set>
21 22 23
#include "cinn/hlir/framework/graph_compiler.h"
#include "cinn/hlir/framework/scope.h"
#include "cinn/runtime/cinn_runtime.h"
24
#include "cinn/runtime/flags.h"
25
#include "paddle/fluid/framework/data_type.h"
26 27 28 29 30 31 32
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"

namespace paddle {
namespace operators {

33 34 35
constexpr char kX[] = "X";
constexpr char kOutputs[] = "Out";
constexpr char kCompilationKey[] = "compilation_key";
36 37

using LoDTensor = framework::LoDTensor;
38 39 40 41 42 43 44
using CinnTensor = ::cinn::hlir::framework::Tensor;
using CinnScope = ::cinn::hlir::framework::Scope;
using CinnCompiler = framework::paddle2cinn::CinnCompiler;
using CinnCompiledObject = framework::paddle2cinn::CinnCompiledObject;

namespace details {

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
class CinnLaunchContext {
 public:
  explicit CinnLaunchContext(const CinnCompiledObject& compiled_obj);

  // Return whether a Paddle variable used on compiled kernels
  bool IsVariableUsed(const std::string& var_name);

  // Allocate buffer to a Paddle tensor with assginment information from CINN
  void MutableTensorData(const std::string& var_name,
                         const platform::Place& place, LoDTensor* paddle_tensor,
                         bool is_internal_var = false);

  // Assign tensor buffer to input or output variables
  void AssignExternalVariable(const std::string& var_name, LoDTensor* tensor);

  // Assign tensor buffer to internal variables
  void AssignInternalVariable(const std::string& var_name, LoDTensor* tensor);

  // Extract internal variable names from CinnScope
  // by excluding used input and output variables
  std::vector<std::string> GetInternalVariableNames();

  // Finalize all execution arguments and return them
  const std::map<std::string, cinn_pod_value_t>& FinalizeArguments() const;

 private:
  // Get CinnTensor with CINN variable name
  CinnTensor GetCinnTensor(const std::string& var_name);

  // Check whether tensors from Paddle and CINN of the same variable
  // are equivalent in type and dimension
  void CheckTensorEquivalent(const std::string& var_name,
                             const LoDTensor& paddle_tensor,
                             const CinnTensor& cinn_tensor);

  // Share the buffer of a Paddle tensor to CINN by delivering memory address
  // to a cinn_buffer_t object
  std::unique_ptr<cinn_buffer_t> ShareTensorWithCinnBuffer(LoDTensor* tensor);

  // Set an argument with (cinn name)->(paddle tensor) pair
  void SetArgument(const std::string& cinn_name, LoDTensor* paddle_tensor);

 private:
  // a variable name map from paddle to cinn
  const std::unordered_map<std::string, std::string>& paddle2cinn_varmap_;
  // the variable scope of cinn
  const std::shared_ptr<CinnScope> cinn_scope_;

  // all variables used by compiled executable program
  std::unordered_set<std::string> cinn_variable_names_;

  // because a cinn_pod_value_t does not own the cinn_buffer_t object,
  // an extra stroage is necessary to keep the object and it can
  // not be released until runtime program finish  execution.
  std::vector<std::unique_ptr<cinn_buffer_t>> hold_buffers_;

  // name to execution argument
  std::map<std::string, cinn_pod_value_t> name2argument_;
};

105 106 107 108 109 110
// Tranform Paddle place to CINN target
const ::cinn::common::Target& PlaceToCinnTarget(const platform::Place& place);

// Print detailed compilation result of graph for debug
void DebugCinnCompiledResult(const CinnCompiledObject& result);

111 112 113
// Launch cinn to execute compiled executable program and wait done
void LaunchCinnExecution(const CinnCompiledObject& compiled_obj,
                         const CinnLaunchContext& context);
114 115 116

// Set cinn FLAGS (such as FLAGS_cinn_cudnn_deterministic) with paddle's FLAGS.
void SetCinnRuntimeFlags();
117
}  // namespace details
118 119 120 121 122

template <typename DeviceContext, typename T>
class CinnLaunchOpKernel : public framework::OpKernel<T> {
 public:
  void Compute(const framework::ExecutionContext& ctx) const override {
123 124
    const auto& scope = ctx.scope();
    const auto& place = ctx.GetPlace();
125 126 127 128 129 130 131
    // Step 1. Find graph object and prepare input
    PADDLE_ENFORCE_EQ(ctx.HasAttr(kCompilationKey), true,
                      platform::errors::NotFound(
                          "No Attribute(%s) found for CinnLaunchOp operator.",
                          kCompilationKey));
    const auto& compilation_key =
        ctx.template Attr<std::string>(kCompilationKey);
132
    VLOG(4) << "CinnLaunchOp attribute(" << kCompilationKey << ") "
133 134
            << "value:\n"
            << CinnCompiler::GetInstance()->ReadableKey(compilation_key);
135 136

    auto input_variable_names = ctx.InputNames(kX);
137 138 139 140 141 142 143 144
    const auto& input_tensors = ctx.MultiInput<LoDTensor>(kX);
    std::map<std::string, const LoDTensor*> inputs_name2tensor;
    std::transform(input_variable_names.begin(), input_variable_names.end(),
                   input_tensors.begin(),
                   std::inserter(inputs_name2tensor, inputs_name2tensor.end()),
                   [](const std::string& name, const LoDTensor* tensor) {
                     return std::make_pair(name, tensor);
                   });
145 146

    // Step 2. Get compilation result of the graph
147
    auto target = details::PlaceToCinnTarget(place);
148 149
    const auto& cinn_compiled_object = CinnCompiler::GetInstance()->Compile(
        compilation_key, inputs_name2tensor, target);
150
    details::DebugCinnCompiledResult(cinn_compiled_object);
151

152 153 154 155 156 157 158
    auto launch_context =
        std::make_unique<details::CinnLaunchContext>(cinn_compiled_object);

    // Step 3. Prepare arguments needed for the compiled executable program.
    VLOG(4) << "CinnLaunchOp prepare arguments";

    // 3.1 Prepare input variables: tensors of input variables have
159 160
    //     been initialized before graph compiled, just check the
    //     equiality between tensors of paddle and cinn.
161 162 163 164 165 166 167 168 169 170 171
    for (const auto& var_name : input_variable_names) {
      if (!launch_context->IsVariableUsed(var_name)) {
        // some input variables don't need for cinn because they are
        // eliminated by optimized passes or some cinn operators use
        // less variables
        VLOG(4) << "Input variable(" << var_name << ") not used by cinn";
        continue;
      }

      launch_context->AssignExternalVariable(
          var_name, scope.GetVar(var_name)->GetMutable<LoDTensor>());
172 173 174
    }

    // 3.2 Prepare output variables: all output variables should
175
    //     be initialized and allocated buffer before
176
    //     the runtime program start execution, the compilation result
177 178
    //     includes details of their buffer assginment and we use that to
    //     allocate space in Paddle. For those variables allocated yet,
179 180
    //     like persistable parameters, just check the equiality between
    //     Paddle allocation and CINN buffer assginment.
181
    auto output_variable_names = ctx.OutputNames(kOutputs);
182 183 184 185 186
    for (const auto var_name : output_variable_names) {
      PADDLE_ENFORCE_EQ(launch_context->IsVariableUsed(var_name), true,
                        platform::errors::InvalidArgument(
                            "Output variable(%s) not used by cinn", var_name));

187
      auto* tensor = scope.GetVar(var_name)->GetMutable<LoDTensor>();
188 189
      if (!tensor->IsInitialized()) {
        launch_context->MutableTensorData(var_name, place, tensor);
190
      }
191 192
      launch_context->AssignExternalVariable(
          var_name, scope.GetVar(var_name)->GetMutable<LoDTensor>());
193 194
    }

195 196 197 198 199 200
    // 3.3 Prepare internal or temporary variables: Create a temporary
    //     scope to keep internal variables within graph or temporary
    //     variables needed by the compiled runtime program in addition.
    //     Here we directly use the names from CinnScope as Paddle variable
    //     names, because they will not be used outside the graph
    //     and should be destructed after computation finished.
201
    auto internal_variable_names = launch_context->GetInternalVariableNames();
202
    auto temp_scope = scope.NewTmpScope();
203 204 205 206
    for (const auto& var_name : internal_variable_names) {
      auto* tensor = temp_scope->Var(var_name)->GetMutable<LoDTensor>();
      launch_context->MutableTensorData(var_name, place, tensor, true);
      launch_context->AssignInternalVariable(var_name, tensor);
207
    }
208

209 210 211 212
    // Step 4. Set CINN runtime FLAGS, such as FLAGS_cinn_cudnn_deterministic.
    details::SetCinnRuntimeFlags();

    // Step 5. Launch CINN to execute the compiled executable program
213
    details::LaunchCinnExecution(cinn_compiled_object, *launch_context);
214
    VLOG(4) << "CinnLaunchOp launch execution done.";
215 216 217 218 219
  }
};

}  // namespace operators
}  // namespace paddle