operator.h 16.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#pragma once

D
dongzhihong 已提交
17
#include <algorithm>
18
#include <atomic>
L
luotao1 已提交
19
#include <memory>
Q
Qiao Longfei 已提交
20
#include <string>
D
dzhwinter 已提交
21
#include <tuple>
Q
Qiao Longfei 已提交
22
#include <unordered_map>
L
luotao1 已提交
23
#include <utility>
Q
Qiao Longfei 已提交
24 25
#include <vector>

Y
Yu Yang 已提交
26
#include "glog/logging.h"  // For VLOG
Y
Yi Wang 已提交
27 28 29 30 31 32
#include "paddle/fluid/framework/attribute.h"
#include "paddle/fluid/framework/block_desc.h"
#include "paddle/fluid/framework/framework.pb.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/op_kernel_type.h"
X
polish  
Xin Pan 已提交
33
#include "paddle/fluid/framework/operator_kernel_configs.h"
Y
Yi Wang 已提交
34 35 36 37 38
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/framework/selected_rows.h"
#include "paddle/fluid/framework/tensor.h"
#include "paddle/fluid/platform/device_context.h"
#include "paddle/fluid/platform/variant.h"
Q
Qiao Longfei 已提交
39

Q
Qiao Longfei 已提交
40 41
DECLARE_int32(inner_op_parallelism);

Q
Qiao Longfei 已提交
42 43 44
namespace paddle {
namespace framework {

45
/// If a variable is a empty variable, that name will be used.
46
constexpr char kEmptyVarName[] = "@EMPTY@";
47 48 49

/// If a variable is a temporary variable, that name will be set in Python,
/// but it will be convert to a unique name in scope after OpCreator.
50
constexpr char kTempVarName[] = "@TEMP@";
51 52 53 54

/// If a variable's name has a certain suffix, it means that the
/// variable is the gradient of another varibale.
/// e.g. Variable "x@GRAD" is the gradient of varibale "x".
55
constexpr char kGradVarSuffix[] = "@GRAD";
56

M
minqiyang 已提交
57 58
constexpr size_t kGradVarSuffixSize = 5U;

59
/// Variables with this suffix are supposed to be filled up with zeros.
60
constexpr char kZeroVarSuffix[] = "@ZERO";
61

C
chengduo 已提交
62 63 64
/// Variables with this suffix are the new Gradient.
constexpr char kNewGradSuffix[] = "@NEWGRAD@";

L
luotao1 已提交
65 66 67 68 69 70 71 72 73
/// If an Op has this attribute, all its kernels should calculate output
/// variable's shape in the corresponding Compute() function. And
/// OperatorWithKernel::RunImpl() would skip call this Op's InferShape()
/// function in its runtime for speedup.
/// TODO(luotao): Note that this temporal attribute would be deleted after all
/// ops contain it.
constexpr char kAllKernelsMustComputeRuntimeShape[] =
    "@ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE@";

D
dzhwinter 已提交
74
// define some kernel priority
75
/* Define multiple kernel type fallback order*/
D
dzhwinter 已提交
76 77
extern std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority;

78
inline std::string GradVarName(const std::string& var_name) {
M
minqiyang 已提交
79 80 81 82 83
  std::string result;
  result.reserve(var_name.size() + kGradVarSuffixSize);
  result += var_name;
  result += kGradVarSuffix;
  return result;
84 85
}

M
minqiyang 已提交
86
inline std::string GradOriginalVarName(const std::string& grad_var_name) {
M
minqiyang 已提交
87
  std::size_t pos = grad_var_name.rfind(kGradVarSuffix);
88 89 90 91 92
  if (pos == std::string::npos) {
    return grad_var_name;
  } else {
    return grad_var_name.substr(0, pos);
  }
93 94
}

Q
qiaolongfei 已提交
95
proto::VarType::Type GetDataTypeOfVar(const Variable* var);
C
chengduo 已提交
96 97
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var);
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var);
Q
qiaolongfei 已提交
98

Q
Qiao Longfei 已提交
99
class OperatorBase;
100
class ExecutionContext;
101

X
Xin Pan 已提交
102 103
class RuntimeContext {
 public:
X
Xin Pan 已提交
104 105
  RuntimeContext(const VariableNameMap& innames,
                 const VariableNameMap& outnames, const Scope& scope);
X
Xin Pan 已提交
106

X
Xin Pan 已提交
107 108 109 110
  RuntimeContext(const VariableValueMap& invars,
                 const VariableValueMap& outvars)
      : inputs(invars), outputs(outvars) {}

X
Xin Pan 已提交
111 112 113 114
  VariableValueMap inputs;
  VariableValueMap outputs;
};

Q
Qiao Longfei 已提交
115
/**
X
Xin Pan 已提交
116
 * OperatorBase has the basic elements that Net will call to do computation.
Q
Qiao Longfei 已提交
117 118 119 120 121 122
 * Only CreateOperator from OpRegistry will new Operator directly. User
 * should always construct a proto message OpDesc and call
 * OpRegistry::CreateOp(op_desc) to get an Operator instance.
 */
class OperatorBase {
 public:
Y
Yu Yang 已提交
123 124
  OperatorBase(const std::string& type, const VariableNameMap& inputs,
               const VariableNameMap& outputs, const AttributeMap& attrs);
125

Q
Qiao Longfei 已提交
126 127
  virtual ~OperatorBase() {}

128
  /// Executor will call this interface function to Run an op.
129 130
  //  The implementation should be written at RunImpl
  void Run(const Scope& scope, const platform::Place& place);
Y
Yu Yang 已提交
131

T
typhoonzero 已提交
132 133 134
  // FIXME(typhoonzero): this is only used for recv_op to stop event_loop.
  virtual void Stop() {}

135 136 137
  /// if scope is not null, also show dimensions of arguments
  virtual std::string DebugStringEx(const Scope* scope) const;
  std::string DebugString() const { return DebugStringEx(nullptr); }
Y
Yu Yang 已提交
138

139 140
  virtual bool SupportGPU() const { return false; }

141 142
  const std::string& Type() const { return type_; }

M
Michal Gallus 已提交
143
  bool HasAttr(const std::string& name) const { return attrs_.count(name); }
144 145
  template <typename T>
  inline const T& Attr(const std::string& name) const {
M
minqiyang 已提交
146 147
    PADDLE_ENFORCE(attrs_.find(name) != attrs_.end(),
                   "%s should be in AttributeMap", name);
148 149 150
    return boost::get<T>(attrs_.at(name));
  }
  const AttributeMap& Attrs() const { return attrs_; }
D
dongzhihong 已提交
151

Y
Yu Yang 已提交
152 153
  const VariableNameMap& Inputs() const { return inputs_; }
  const VariableNameMap& Outputs() const { return outputs_; }
154

155
  bool HasInputs(const std::string& name) const;
Y
Yu Yang 已提交
156
  //! Get a input with argument's name described in `op_proto`
157
  std::string Input(const std::string& name) const;
Y
Yu Yang 已提交
158
  //! Get a input which has multiple variables.
Y
Yu Yang 已提交
159
  const std::vector<std::string>& Inputs(const std::string& name) const;
160
  //! Get all inputs variable names
Q
qijun 已提交
161 162
  std::vector<std::string> InputVars() const;

163
  bool HasOutputs(const std::string& name) const;
Y
Yu Yang 已提交
164
  //! Get a output with argument's name described in `op_proto`
165
  std::string Output(const std::string& name) const;
Y
Yu Yang 已提交
166 167
  //! Get an output which has multiple variables.
  //! TODO add a vector_view to prevent memory copy.
Y
Yu Yang 已提交
168
  const std::vector<std::string>& Outputs(const std::string& name) const;
169
  //! Get all outputs variable names
Y
Yu Yang 已提交
170
  virtual std::vector<std::string> OutputVars(bool has_intermediate) const;
171

172
  void SetIsCalledByExecutor(bool x) { run_by_executor_ = x; }
B
baojun-nervana 已提交
173
  virtual void RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
174 175
                                 const platform::Place& place,
                                 const RuntimeContext& ctx) const {}
176

Q
qiaolongfei 已提交
177
 protected:
Q
Qiao Longfei 已提交
178
  std::string type_;
D
dongzhihong 已提交
179
  // NOTE: in case of OpGrad, inputs_ contains:
180
  // I (Inputs)
D
dongzhihong 已提交
181 182
  // O (Outputs)
  // OG (Output Gradients)
Y
Yu Yang 已提交
183
  VariableNameMap inputs_;
Y
Yu Yang 已提交
184

D
dongzhihong 已提交
185 186
  // NOTE: in case of OpGrad, outputs_ contains
  // IG (Inputs Gradients)
Y
Yu Yang 已提交
187
  VariableNameMap outputs_;
Q
Qiao Longfei 已提交
188
  AttributeMap attrs_;
189 190
  // Whether this operator executes in an Executor.
  bool run_by_executor_{true};
191 192 193 194

 private:
  void GenerateTemporaryNames();
  void CheckAllInputOutputSet() const;
195 196
  virtual void RunImpl(const Scope& scope,
                       const platform::Place& place) const = 0;
Y
Yan Chunwei 已提交
197 198
};

199 200 201 202 203 204 205 206 207 208 209 210 211
#ifdef PADDLE_WITH_CUDA
using KernelConfig = boost::variant<
    std::shared_ptr<AlgorithmsCache<cudnnConvolutionFwdAlgo_t>>,
    std::shared_ptr<AlgorithmsCache<cudnnConvolutionBwdDataAlgo_t>>,
    std::shared_ptr<AlgorithmsCache<cudnnConvolutionBwdFilterAlgo_t>>>;
#else
using KernelConfig = boost::variant<boost::blank>;
#endif

using OpKernelConfigsMap =
    std::unordered_map<OpKernelType, std::vector<KernelConfig>,
                       OpKernelType::Hash>;

212
class ExecutionContext {
Y
Yan Chunwei 已提交
213
 public:
214
  ExecutionContext(const OperatorBase& op, const Scope& scope,
X
Xin Pan 已提交
215
                   const platform::DeviceContext& device_context,
216 217 218 219 220 221 222
                   const RuntimeContext& ctx,
                   std::vector<KernelConfig>* configs)
      : op_(op),
        scope_(scope),
        device_context_(device_context),
        ctx_(ctx),
        kernel_configs_(configs) {}
223

Q
qiaolongfei 已提交
224 225 226 227
  const OperatorBase& op() const { return op_; }

  const Scope& scope() const { return scope_; }

Q
qiaolongfei 已提交
228
  template <typename T>
Y
Yu Yang 已提交
229 230
  inline const T& Attr(const std::string& name) const {
    return op_.Attr<T>(name);
Q
qiaolongfei 已提交
231 232
  }

233
  bool HasInput(const std::string& name) const;
234

235
  bool HasOutput(const std::string& name) const;
236

Y
Yu Yang 已提交
237
  size_t InputSize(const std::string& name) const {
Y
Yu Yang 已提交
238
    return op_.Inputs(name).size();
Y
Yan Chunwei 已提交
239 240
  }

Y
Yu Yang 已提交
241
  size_t OutputSize(const std::string& name) const {
Y
Yu Yang 已提交
242
    return op_.Outputs(name).size();
Y
Yan Chunwei 已提交
243 244
  }

X
Xin Pan 已提交
245
  const Variable* InputVar(const std::string& name) const;
Y
Yan Chunwei 已提交
246

X
Xin Pan 已提交
247
  Variable* OutputVar(const std::string& name) const;
Y
Yan Chunwei 已提交
248

249 250
  const std::vector<const Variable*> MultiInputVar(
      const std::string& name) const {
X
Xin Pan 已提交
251 252 253 254
    auto it = ctx_.inputs.find(name);
    if (it == ctx_.inputs.end()) {
      return {};
    }
G
Gabor Buella 已提交
255
    return {it->second.begin(), it->second.end()};
X
Xin Pan 已提交
256 257 258 259 260 261 262 263 264 265 266
  }

  std::vector<Variable*> MultiOutputVar(const std::string& name) const {
    auto names = op_.Outputs(name);
    auto it = ctx_.outputs.find(name);
    if (it == ctx_.outputs.end()) {
      return {};
    }
    return it->second;
  }

267 268
  template <typename T>
  const T* Input(const std::string& name) const {
Y
Yu Yang 已提交
269
    auto* var = InputVar(name);
270
    return var == nullptr ? nullptr : &var->Get<T>();
271 272 273 274
  }

  template <typename T>
  T* Output(const std::string& name) const {
275
    auto var = OutputVar(name);
276
    return var == nullptr ? nullptr : var->GetMutable<T>();
277 278 279 280
  }

  template <typename T>
  const std::vector<const T*> MultiInput(const std::string& name) const {
X
Xin Pan 已提交
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
    auto it = ctx_.inputs.find(name);
    if (it == ctx_.inputs.end()) {
      return {};
    }
    const std::vector<Variable*>& vars = it->second;
    std::vector<const T*> res;
    res.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                   [&](Variable* var) -> const T* {
                     return var == nullptr ? nullptr : &var->Get<T>();
                   });
    return res;
  }

  template <typename T>
  std::vector<T*> MultiOutput(const std::string& name) const {
    auto it = ctx_.outputs.find(name);
    if (it == ctx_.outputs.end()) {
      return {};
    }
    const std::vector<Variable*>& vars = it->second;
    std::vector<T*> res;
    res.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                   [&](Variable* var) -> T* {
                     return var == nullptr ? nullptr : var->GetMutable<T>();
                   });
    return res;
  }

311
  platform::Place GetPlace() const { return device_context_.GetPlace(); }
Q
qijun 已提交
312

Q
QI JUN 已提交
313 314 315 316 317
  template <typename DeviceContextType>
  const DeviceContextType& device_context() const {
    return *reinterpret_cast<const DeviceContextType*>(&device_context_);
  }

318
  const platform::DeviceContext& device_context() const {
Q
qijun 已提交
319
    return device_context_;
Q
qijun 已提交
320
  }
Q
qijun 已提交
321

Q
QI JUN 已提交
322 323 324 325 326 327 328 329
#ifdef PADDLE_WITH_CUDA
  const inline platform::CUDADeviceContext& cuda_device_context() const {
    PADDLE_ENFORCE(platform::is_gpu_place(device_context_.GetPlace()));
    return *reinterpret_cast<const platform::CUDADeviceContext*>(
        &device_context_);
  }
#endif

D
dzhwinter 已提交
330
  //! Get actual name vector for this input.
D
Dong Zhihong 已提交
331 332 333
  const std::vector<std::string>& Inputs(const std::string& name) const {
    return op_.Inputs(name);
  }
D
Dong Zhihong 已提交
334

D
dzhwinter 已提交
335
  //! Get actual name vector for this output.
D
Dong Zhihong 已提交
336 337 338 339
  const std::vector<std::string>& Outputs(const std::string& name) const {
    return op_.Outputs(name);
  }

X
Xin Pan 已提交
340 341 342 343 344 345 346 347 348 349 350 351 352 353
  template <typename T, typename DevContext>
  Tensor AllocateTmpTensor(const framework::DDim& dim,
                           const DevContext& dev_ctx) const {
    auto tmp_allocation_ptr = platform::DeviceTemporaryAllocator::Instance()
                                  .Get<DevContext>(dev_ctx)
                                  .Allocate(product(dim) * sizeof(T));
    auto& deleter = tmp_allocation_ptr.get_deleter();
    auto* allocation_ptr = tmp_allocation_ptr.release();
    auto shared_allocation = std::shared_ptr<memory::allocation::Allocation>(
        allocation_ptr, deleter);

    PADDLE_ENFORCE(
        dynamic_cast<platform::TemporaryAllocation*>(allocation_ptr) != nullptr,
        "The AllocationPtr must be TemporaryAllocation.");
354
    PADDLE_ENFORCE_GE(allocation_ptr->size(),
X
Xin Pan 已提交
355 356 357 358 359 360 361 362 363
                      framework::product(dim) * sizeof(T));

    paddle::framework::Tensor temp_tensor(
        framework::ToDataType(std::type_index(typeid(T))));
    temp_tensor.Resize(dim);
    temp_tensor.ResetHolder(std::move(shared_allocation));
    return temp_tensor;
  }

364 365 366 367 368 369 370 371
  template <typename T>
  T& GetKernelConfig(int idx) const {
    PADDLE_ENFORCE(kernel_configs_ && kernel_configs_->size() > idx,
                   "%s selected kernel doesn't have kernel config %lu <= %d",
                   op_.Type().c_str(), kernel_configs_->size(), idx);
    return *boost::get<std::shared_ptr<T>>(kernel_configs_->at(idx));
  }

372
 private:
373 374
  const OperatorBase& op_;
  const Scope& scope_;
375
  const platform::DeviceContext& device_context_;
X
Xin Pan 已提交
376
  const RuntimeContext& ctx_;
377
  mutable std::vector<KernelConfig>* kernel_configs_;
Q
Qiao Longfei 已提交
378 379
};

380 381 382 383 384 385 386 387 388 389 390 391 392 393
template <>
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const;

template <>
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
    const std::string& name) const;

template <>
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const;

template <>
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
    const std::string& name) const;

Y
Yu Yang 已提交
394
class OpKernelBase {
Q
qijun 已提交
395
 public:
Q
qijun 已提交
396
  /**
397
   * ExecutionContext is the only parameter of Kernel Run function.
Q
qijun 已提交
398 399
   * Run will get input/output variables, state such as momentum and
   * device resource such as CUDA stream, cublas handle, etc. from
400
   * ExecutionContext. User should construct it before run the Operator.
Q
qijun 已提交
401 402
   */

403
  virtual void Compute(const ExecutionContext& context) const = 0;
Y
Yu Yang 已提交
404

Y
Yu Yang 已提交
405 406 407 408 409 410 411
  virtual ~OpKernelBase() = default;
};

template <typename T>
class OpKernel : public OpKernelBase {
 public:
  using ELEMENT_TYPE = T;
Y
Yu Yang 已提交
412 413
};

Y
Yu Yang 已提交
414 415
class OperatorWithKernel : public OperatorBase {
 public:
Y
yuyang18 已提交
416
  using OpKernelFunc = std::function<void(const ExecutionContext&)>;
Y
Yu Yang 已提交
417
  using OpKernelMap =
Y
yuyang18 已提交
418
      std::unordered_map<OpKernelType, OpKernelFunc, OpKernelType::Hash>;
Q
Qiao Longfei 已提交
419

Y
Yu Yang 已提交
420 421
  OperatorWithKernel(const std::string& type, const VariableNameMap& inputs,
                     const VariableNameMap& outputs, const AttributeMap& attrs)
Y
Yu Yang 已提交
422 423
      : OperatorBase(type, inputs, outputs, attrs) {}

Y
Yu Yang 已提交
424 425 426 427
  static std::unordered_map<std::string /* op_type */, OpKernelMap>&
  AllOpKernels() {
    static std::unordered_map<std::string, OpKernelMap> g_all_op_kernels;
    return g_all_op_kernels;
Y
Yu Yang 已提交
428
  }
Y
Yan Chunwei 已提交
429

430
  bool SupportGPU() const override {
Y
Yu Yang 已提交
431 432 433 434 435
    auto& op_kernels = OperatorWithKernel::AllOpKernels().at(type_);
    return std::any_of(op_kernels.begin(), op_kernels.end(),
                       [](OpKernelMap::const_reference kern_pair) {
                         return platform::is_gpu_place(kern_pair.first.place_);
                       });
436 437
  }

438 439 440
  virtual void InferShape(InferShapeContext* ctx) const {
    OpInfoMap::Instance().Get(Type()).infer_shape_(ctx);
  }
Y
Yu Yang 已提交
441

X
Xin Pan 已提交
442 443
  void RuntimeInferShape(const Scope& scope, const platform::Place& place,
                         const RuntimeContext& ctx) const override;
B
baojun-nervana 已提交
444

445
  virtual OpKernelType GetExpectedKernelType(const ExecutionContext& ctx) const;
X
Xin Pan 已提交
446

X
polish  
Xin Pan 已提交
447 448
  std::vector<KernelConfig>* GetKernelConfig(const OpKernelType& key) const;

X
Xin Pan 已提交
449
 protected:
450 451 452
  virtual OpKernelType GetKernelTypeForVar(
      const std::string& var_name, const Tensor& tensor,
      const OpKernelType& expected_kernel_type) const;
Y
Yu Yang 已提交
453 454

 private:
455
  // indicate kernel DataType by input data. By default all input data must be
Y
Yu Yang 已提交
456
  // same.
457
  proto::VarType::Type IndicateDataType(const ExecutionContext& ctx) const;
458
  void RunImpl(const Scope& scope, const platform::Place& place) const final;
Y
yuyang18 已提交
459 460 461 462 463 464 465

  /**
   * Transfer data from scope to a transfered scope. If there is no data need to
   * be tranfered, it returns nullptr.
   *
   * * transfered_inplace_vars is a output vector.
   */
X
Xin Pan 已提交
466 467 468 469
  Scope* PrepareData(const Scope& scope,
                     const OpKernelType& expected_kernel_key,
                     std::vector<std::string>* transfered_inplace_vars,
                     RuntimeContext* ctx) const;
Y
yuyang18 已提交
470 471 472 473

  void TransferInplaceVarsBack(const Scope& scope,
                               const std::vector<std::string>& inplace_vars,
                               const Scope& exec_scope) const;
474 475 476

 protected:
  mutable OpKernelConfigsMap kernel_configs_map_;
Q
Qiao Longfei 已提交
477 478
};

Y
Yu Yang 已提交
479 480
extern bool OpSupportGPU(const std::string& op_type);

Q
Qiao Longfei 已提交
481 482
}  // namespace framework
}  // namespace paddle