operator.cc 60.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include "paddle/fluid/framework/operator.h"

17
#include <glog/logging.h>
P
peizhilin 已提交
18 19
#include <sstream>
#include <string>
20

21
#include "gflags/gflags.h"
Y
Yi Wang 已提交
22
#include "paddle/fluid/framework/data_transform.h"
23
#include "paddle/fluid/framework/data_type_transform.h"
W
WangXi 已提交
24
#include "paddle/fluid/framework/details/nan_inf_utils.h"
25
#include "paddle/fluid/framework/op_call_stack.h"
Y
Yi Wang 已提交
26
#include "paddle/fluid/framework/shape_inference.h"
27
#include "paddle/fluid/framework/transfer_scope_cache.h"
28
#include "paddle/fluid/framework/unused_var_check.h"
Y
Yi Wang 已提交
29
#include "paddle/fluid/framework/var_type.h"
L
Leo Chen 已提交
30
#include "paddle/fluid/platform/enforce.h"
31
#include "paddle/fluid/platform/profiler.h"
32 33 34 35 36 37

namespace paddle {
namespace framework {
class LoDTensor;
}  // namespace framework
}  // namespace paddle
38 39 40
#ifdef PADDLE_WITH_XPU
#include "paddle/fluid/platform/xpu_info.h"
#endif
Q
Qiao Longfei 已提交
41

42 43 44 45
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/platform/mkldnn_helper.h"
#endif

D
dzhwinter 已提交
46
DECLARE_bool(benchmark);
47
DECLARE_bool(check_nan_inf);
48
DECLARE_bool(enable_unused_var_check);
Q
Qiao Longfei 已提交
49
DEFINE_int32(inner_op_parallelism, 0, "number of threads for inner op");
D
dzhwinter 已提交
50

Q
Qiao Longfei 已提交
51 52 53
namespace paddle {
namespace framework {

54 55 56 57 58 59
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
60

61 62
static DDim GetDimsDebug(const Scope& scope, const std::string& name,
                         bool get_actual_dim = false) {
63
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
64 65
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
66 67
  }

M
minqiyang 已提交
68 69 70 71 72 73 74 75 76
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
77 78 79 80 81
  } else {
    return DDim({-1});
  }
}

Q
Qiao Longfei 已提交
82 83 84 85 86 87
static bool VarInited(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

D
dzhwinter 已提交
88 89 90 91 92
static std::string GetDtype(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
93

M
minqiyang 已提交
94 95 96
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
97 98
      return "";
    }
Y
Yu Yang 已提交
99
    return DataTypeToString(tensor.type());
M
minqiyang 已提交
100
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
101 102 103 104
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
Y
Yu Yang 已提交
105
      return DataTypeToString(tensor.type());
Q
Qiao Longfei 已提交
106
    }
D
dzhwinter 已提交
107 108 109 110 111
  } else {
    return "";
  }
}

L
Leo Chen 已提交
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
static std::string GetPlace(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
  auto to_string = [](const platform::Place& p) {
    std::stringstream sstream;
    sstream << p;
    return sstream.str();
  };

  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "";
    }
    return to_string(tensor.place());
  } else if (var->IsType<SelectedRows>()) {
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
      return to_string(tensor.place());
    }
  } else {
    return "";
  }
}

141 142 143 144 145 146
static int GetRowSize(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
147 148
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
149 150 151 152 153
  }

  return -1;
}

154
static LoD GetLoDDebug(const Scope& scope, const std::string& name) {
Q
Qiao Longfei 已提交
155 156 157 158 159 160 161
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
162 163 164
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.lod();
Q
Qiao Longfei 已提交
165 166 167 168 169
  } else {
    return default_lod;
  }
}

X
Xin Pan 已提交
170 171 172 173 174
RuntimeContext::RuntimeContext(const VariableNameMap& innames,
                               const VariableNameMap& outnames,
                               const Scope& scope) {
  for (auto& var_name_item : innames) {
    std::vector<Variable*>& input_vars = inputs[var_name_item.first];
X
Xin Pan 已提交
175
    input_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
176 177 178 179 180 181
    for (auto& var_name : var_name_item.second) {
      input_vars.push_back(scope.FindVar(var_name));
    }
  }
  for (auto& var_name_item : outnames) {
    std::vector<Variable*>& output_vars = outputs[var_name_item.first];
X
Xin Pan 已提交
182
    output_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
183 184 185 186 187 188
    for (auto& var_name : var_name_item.second) {
      output_vars.push_back(scope.FindVar(var_name));
    }
  }
}

189
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
P
peizhilin 已提交
190 191 192
  try {
    VLOG(4) << place << " " << DebugStringEx(&scope);
    if (platform::is_gpu_place(place)) {
193
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
194 195 196 197
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with CUDA support.",
          place));
198
#else
199
      auto dev_id = BOOST_GET_CONST(platform::CUDAPlace, place).device;
P
peizhilin 已提交
200
      platform::SetDeviceId(dev_id);
201 202 203
#endif
    } else if (platform::is_xpu_place(place)) {
#ifndef PADDLE_WITH_XPU
204 205 206 207
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with XPU support.",
          place));
208 209 210
#else
      auto dev_id = BOOST_GET_CONST(platform::XPUPlace, place).device;
      platform::SetXPUDeviceId(dev_id);
211 212 213 214 215 216 217 218 219 220
#endif
    } else if (platform::is_npu_place(place)) {
#ifndef PADDLE_WITH_ASCEND_CL
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with NPU support.",
          place));
#else
      auto dev_id = BOOST_GET_CONST(platform::NPUPlace, place).device;
      platform::SetNPUDeviceId(dev_id);
221
#endif
P
peizhilin 已提交
222
    }
P
peizhilin 已提交
223

224
    {
225 226 227 228 229 230
      // TODO(wangchaochaohu) : refine code to use only one RecordEvent)
      // in order to record different op type cost time
      // and different op name cost time,we set two event.
      platform::RecordEvent op_type_record_event(Type());
      auto op_name = platform::OpName(outputs_, Type());
      platform::RecordEvent op_name_record_event(
231
          op_name, platform::EventRole::kUniqueOp);
P
peizhilin 已提交
232 233
      RunImpl(scope, place);
    }
234

Z
Zhang Ting 已提交
235
    VLOG(3) << GetExecutionPlace(place) << " " << DebugStringEx(&scope);
236
  } catch (platform::EnforceNotMet& exception) {
237
    framework::InsertCallStackInfo(Type(), Attrs(), &exception);
238
    throw std::move(exception);
239 240 241 242 243 244
  } catch (platform::EOFException&) {
    std::rethrow_exception(std::current_exception());
  } catch (std::exception& ex) {
    LOG(WARNING) << Type() << " raises an exception "
                 << platform::demangle(typeid(ex).name()) << ", " << ex.what();
    std::rethrow_exception(std::current_exception());
P
peizhilin 已提交
245
  } catch (...) {
246
    LOG(WARNING) << Type() << " raises an unknown exception";
P
peizhilin 已提交
247
    std::rethrow_exception(std::current_exception());
248
  }
249 250
}

251
bool OperatorBase::HasInputs(const std::string& name) const {
M
minqiyang 已提交
252
  return inputs_.find(name) != inputs_.end();
253 254
}

255
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
256
  auto& ins = Inputs(name);
257 258
  PADDLE_ENFORCE_LE(
      ins.size(), 1UL,
259
      platform::errors::InvalidArgument(
260 261
          "Operator %s's input %s should contain only one variable.", type_,
          name));
Y
Yu Yang 已提交
262
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
263 264
}

Y
Yu Yang 已提交
265 266
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
267
  auto it = inputs_.find(name);
268 269 270 271
  PADDLE_ENFORCE_NE(
      it, inputs_.end(),
      platform::errors::NotFound("Operator %s does not have the input %s.",
                                 type_, name));
Y
Yu Yang 已提交
272
  return it->second;
Y
Yan Chunwei 已提交
273 274
}

275
bool OperatorBase::HasOutputs(const std::string& name) const {
276
  if (outputs_.find(name) != outputs_.end()) {
277 278 279 280 281 282
    return true;
  } else {
    return false;
  }
}

283
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
284
  auto& outs = Outputs(name);
285 286 287 288 289
  PADDLE_ENFORCE_LE(
      outs.size(), 1UL,
      platform::errors::InvalidArgument(
          "Operator %s's output %s should contain only one variable.", type_,
          name));
Y
Yu Yang 已提交
290
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
291 292
}

Y
Yu Yang 已提交
293 294
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
295
  auto it = outputs_.find(name);
296 297 298 299
  PADDLE_ENFORCE_NE(
      it, outputs_.end(),
      platform::errors::NotFound(
          "Operator %s does not have an output called %s.", type_, name));
Y
Yu Yang 已提交
300
  return it->second;
Y
Yan Chunwei 已提交
301 302
}

303
std::string OperatorBase::DebugStringEx(const Scope* scope) const {
Q
Qiao Longfei 已提交
304
  std::stringstream ss;
Y
Yu Yang 已提交
305
  ss << "Op(" << type_ << "), inputs:{";
306

307
  const std::unordered_set<std::string>* no_need_buffer_vars = nullptr;
308 309
  if (info_ && info_->NoNeedBufferVarsInferer()) {
    no_need_buffer_vars =
310 311
        &(Info().NoNeedBufferVarsInferer()(Inputs(), Outputs(), Attrs()));
    if (no_need_buffer_vars->empty()) no_need_buffer_vars = nullptr;
312 313
  }

Y
Yu Yang 已提交
314 315
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
316 317
    bool is_no_need_buffer_var =
        (no_need_buffer_vars && no_need_buffer_vars->count(input.first) > 0);
Y
Yu Yang 已提交
318 319
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
320 321
      auto var_name = input.second[i];
      ss << var_name;
322
      if (scope) {
Q
Qiao Longfei 已提交
323 324 325 326 327 328 329
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
330 331 332
          std::string dtype = is_no_need_buffer_var
                                  ? "unknown_dtype"
                                  : GetDtype(*scope, var_name);
Q
Qiao Longfei 已提交
333
          ss << ":" << dtype;
334 335
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
L
Leo Chen 已提交
336
          ss << "(" << GetPlace(*scope, var_name) << ")";
337
        }
338
      }
Y
Yu Yang 已提交
339 340 341
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
342
    }
Y
Yu Yang 已提交
343
    ss << "]";
Y
Yu Yang 已提交
344 345
    ++it;
    if (it != inputs_.end()) {
346 347
      ss << ", ";
    }
Q
Qiao Longfei 已提交
348
  }
Y
Yu Yang 已提交
349
  ss << "}, outputs:{";
Y
Yu Yang 已提交
350 351
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
352 353
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
354 355
      auto var_name = output.second[i];
      ss << var_name;
356
      if (scope) {
Q
Qiao Longfei 已提交
357 358 359 360 361 362 363
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
364 365
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
366 367
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
L
Leo Chen 已提交
368
          ss << "(" << GetPlace(*scope, var_name) << ")";
369
        }
370
      }
Y
Yu Yang 已提交
371 372 373
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
374
    }
Y
Yu Yang 已提交
375
    ss << "]";
Y
Yu Yang 已提交
376 377
    ++it;
    if (it != outputs_.end()) {
378 379
      ss << ", ";
    }
Q
Qiao Longfei 已提交
380
  }
Y
Yu Yang 已提交
381
  ss << "}.";
Q
Qiao Longfei 已提交
382 383 384
  return ss.str();
}

Y
Yu Yang 已提交
385
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
386 387
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
388
                           const AttributeMap& attrs)
S
sneaxiy 已提交
389 390 391 392 393 394
    : type_(type),
      inputs_(inputs),
      outputs_(outputs),
      attrs_(attrs),
      // NOTE(zjl): why op_info may be nullptr?
      info_(OpInfoMap::Instance().GetNullable(type)) {
H
hong 已提交
395 396 397 398 399 400 401 402
  // In dygraph mode, all the OperatorBase will be constructed by function:
  // framework::OpRegistry::CreateOp(type, {}, {}, {}, false).
  // Inputs, outputs and attrs will be set to empty map
  // to improve the execution efficiency of dygraph.
  if (inputs_.size() > 0 || outputs_.size() > 0) {
    GenerateTemporaryNames();
    CheckAllInputOutputSet();
  }
Y
Yu Yang 已提交
403
}
404

Q
qijun 已提交
405 406
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
407
  for (auto& o : inputs_) {
Q
qijun 已提交
408 409 410 411 412 413
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
414 415 416 417 418 419 420 421 422 423
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
S
sneaxiy 已提交
424
  auto& info = Info();
Y
Yu Yang 已提交
425 426

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
427
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
428 429 430 431 432 433 434 435 436
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
437 438
}

439
void OperatorBase::CheckAllInputOutputSet() const {
S
sneaxiy 已提交
440
  if (info_ == nullptr || info_->proto_ == nullptr) return;
441

S
sneaxiy 已提交
442
  for (auto& in : info_->Proto().inputs()) {
443
    if (!in.dispensable()) {
444 445 446 447
      PADDLE_ENFORCE_NE(
          inputs_.find(in.name()), inputs_.end(),
          platform::errors::NotFound("Operator %s's input (%s) is not set.",
                                     Type(), in.name()));
448
    }
449 450
  }

S
sneaxiy 已提交
451
  for (auto& out : info_->Proto().outputs()) {
452
    if (!out.dispensable()) {
453 454 455 456
      PADDLE_ENFORCE_NE(
          outputs_.find(out.name()), outputs_.end(),
          platform::errors::NotFound("Operator %s's output (%s) is not set.",
                                     Type(), out.name()));
457
    }
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

B
baojun-nervana 已提交
474
static bool VarIsTensor(const Variable& var) {
C
chengduo 已提交
475
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
476 477
}

C
chengduo 已提交
478
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
479 480 481 482
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
483
  } else {
484 485 486
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Variable type is %s, expect LoDTensor or SelectedRows.",
        ToTypeName(var.Type())));
Q
QI JUN 已提交
487 488 489
  }
}

C
chengduo 已提交
490
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
491
  if (var->IsType<LoDTensor>()) {
492
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
493
  } else if (var->IsType<SelectedRows>()) {
494
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
495
  } else {
496 497 498
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Variable type is %s, expect LoDTensor or SelectedRows.",
        ToTypeName(var->Type())));
Q
QI JUN 已提交
499 500 501
  }
}

502
bool ExecutionContext::HasInput(const std::string& name) const {
503
  auto* var = InputVar(name);
504 505 506 507
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
508
  auto* var = OutputVar(name);
509 510 511
  return var != nullptr;
}

X
Xin Pan 已提交
512
const Variable* ExecutionContext::InputVar(const std::string& name) const {
513 514
  LogVarUsageIfUnusedVarCheckEnabled(name);

X
Xin Pan 已提交
515 516 517
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) return nullptr;

518 519
  PADDLE_ENFORCE_LE(
      it->second.size(), 1UL,
520
      platform::errors::InvalidArgument(
521 522
          "Operator %s's input %s should contain only one variable.",
          op_.Type(), name));
X
Xin Pan 已提交
523 524 525
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
526
Variable* ExecutionContext::OutputVar(const std::string& name) const {
X
Xin Pan 已提交
527 528 529
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) return nullptr;

530 531 532 533 534
  PADDLE_ENFORCE_LE(
      it->second.size(), 1UL,
      platform::errors::InvalidArgument(
          "Operator %s's output %s should contain only one variable.",
          op_.Type(), name));
X
Xin Pan 已提交
535 536 537
  return it->second.empty() ? nullptr : it->second[0];
}

538
template <>
539
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
540
  return Input<LoDTensor>(name);
541 542 543
}

template <>
544
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
545
    const std::string& name) const {
546 547
  LogVarUsageIfUnusedVarCheckEnabled(name);

H
hong 已提交
548 549
  auto vars = MultiInputVar(name);
  if (vars.size() == 0) {
X
Xin Pan 已提交
550 551 552 553 554
    return {};
  }
  std::vector<const Tensor*> res;
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
H
hong 已提交
555
                 [&](const Variable* var) -> const Tensor* {
X
Xin Pan 已提交
556
                   if (var == nullptr) return nullptr;
557 558 559 560 561
                   PADDLE_ENFORCE_EQ(var->IsType<LoDTensor>(), true,
                                     platform::errors::InvalidArgument(
                                         "Input variable should be LoDTensor, "
                                         "but the received type is %s.",
                                         ToTypeName(var->Type())));
X
Xin Pan 已提交
562 563 564 565 566
                   return &(var->Get<LoDTensor>());
                 });
  return res;
}

567
template <>
568
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
569
  return Output<LoDTensor>(name);
570 571 572
}

template <>
573
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
574
    const std::string& name) const {
H
hong 已提交
575 576 577
  auto vars = MultiOutputVar(name);

  if (vars.size() == 0) {
578 579
    return {};
  }
580
  std::vector<Tensor*> res;
581 582 583 584 585
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                 [&](Variable* var) -> Tensor* {
                   return var == nullptr ? nullptr
                                         : var->GetMutable<LoDTensor>();
586
                 });
587 588 589
  return res;
}

Y
Yu Yang 已提交
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

605 606
class RuntimeInferShapeContext : public InferShapeContext {
 public:
607
  RuntimeInferShapeContext(const OperatorBase& op, const RuntimeContext& ctx)
G
Gabor Buella 已提交
608
      : op_(op), ctx_(ctx) {}
609 610

  bool HasInput(const std::string& name) const override {
611
    // has only one input
X
Xin Pan 已提交
612
    const auto& ins = ctx_.inputs;
613 614
    auto it = ins.find(name);
    if (it == ins.end()) {
615 616
      return false;
    }
617
    const auto& in = it->second;
X
Xin Pan 已提交
618
    if (in.size() == 0) return false;
619 620 621 622
    PADDLE_ENFORCE_EQ(
        in.size(), 1UL,
        platform::errors::InvalidArgument(
            "Input %s should not contain more than one inputs.", name));
X
Xin Pan 已提交
623
    return in[0] != nullptr;
624 625 626
  }

  bool HasOutput(const std::string& name) const override {
627
    // has only one output
X
Xin Pan 已提交
628
    const auto& outs = ctx_.outputs;
629 630
    auto it = outs.find(name);
    if (it == outs.end()) {
631 632
      return false;
    }
633
    const auto& out = it->second;
X
Xin Pan 已提交
634
    if (out.size() == 0) {
635 636
      return false;
    }
637 638 639 640
    PADDLE_ENFORCE_EQ(
        out.size(), 1UL,
        platform::errors::InvalidArgument(
            "Output %s should not contain more than one outputs.", name));
X
Xin Pan 已提交
641
    return out[0] != nullptr;
642 643 644
  }

  bool HasInputs(const std::string& name) const override {
X
Xin Pan 已提交
645 646
    const auto& ins = ctx_.inputs;
    auto it = ins.find(name);
X
fix  
Xin Pan 已提交
647
    if (it == ins.end() || it->second.empty()) {
648 649
      return false;
    }
X
Xin Pan 已提交
650 651
    for (auto& input : it->second) {
      if (input == nullptr) {
652 653 654 655 656 657 658
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
X
Xin Pan 已提交
659 660
    const auto& outs = ctx_.outputs;
    auto it = outs.find(name);
X
fix  
Xin Pan 已提交
661
    if (it == outs.end() || it->second.empty()) {
662 663
      return false;
    }
X
Xin Pan 已提交
664 665
    for (auto& output : it->second) {
      if (output == nullptr) {
666 667 668 669 670 671 672 673
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

H
hong 已提交
674
  std::vector<std::string> Inputs(const std::string& name) const override {
675 676 677
    return op_.Inputs(name);
  }

H
hong 已提交
678
  std::vector<std::string> Outputs(const std::string& name) const override {
679 680 681
    return op_.Outputs(name);
  }

682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
  std::string GetInputNameByIdx(size_t idx) const override {
    auto& op_proto =
        paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_;
    PADDLE_ENFORCE_LT(idx, op_proto->inputs().size(),
                      platform::errors::OutOfRange(
                          "The index should be less than the size of inputs of "
                          "operator %s, but got index is %d and size is %d",
                          op_.Type(), idx, op_proto->inputs().size()));
    return op_proto->inputs()[idx].name();
  }

  std::string GetOutputNameByIdx(size_t idx) const override {
    auto& op_proto =
        paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_;
    PADDLE_ENFORCE_LT(
        idx, op_proto->outputs().size(),
        platform::errors::OutOfRange(
            "The index should be less than the size of outputs of "
            "operator %s, but got index is %d and size is %d",
            op_.Type(), idx, op_proto->outputs().size()));
    return op_proto->outputs()[idx].name();
  }

705 706
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
X
Xin Pan 已提交
707 708
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
    PADDLE_ENFORCE_NE(
        in_it, ctx_.inputs.end(),
        platform::errors::NotFound("Input %s does not exist.", in));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output %s does not exist.", out));
    PADDLE_ENFORCE_LT(i, in_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of input dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          in_it->second.size(), i));
    PADDLE_ENFORCE_LT(j, out_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of output dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          out_it->second.size(), j));
X
Xin Pan 已提交
725 726 727

    Variable* in_var = in_it->second[i];
    Variable* out_var = out_it->second[j];
728

729 730 731 732 733
    PADDLE_ENFORCE_EQ(
        in_var->Type(), out_var->Type(),
        platform::errors::InvalidArgument(
            "The type of input (%s) and output (%s) are inconsistent.", in,
            out));
734 735 736 737 738 739 740 741 742 743 744 745

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
746
      PADDLE_THROW(platform::errors::Unimplemented(
747
          "Currently, the input type of ShareDim only can be LoDTensor "
748
          "or SelectedRows."));
749 750 751
    }
  }

H
hong 已提交
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
  void ShareAllLoD(const std::string& in,
                   const std::string& out) const override {
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(),
                      platform::errors::NotFound(
                          "Input [%s] found error in Op [%s]", in, op_.Type()));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output [%s] found error in Op [%s]", out,
                                   op_.Type()));

    auto& in_var_list = in_it->second;
    auto& out_var_list = out_it->second;

    PADDLE_ENFORCE_EQ(
        in_var_list.size(), out_var_list.size(),
        platform::errors::PreconditionNotMet(
T
tianshuo78520a 已提交
770
            "Op [%s]: Input var size should be equal with output var size",
H
hong 已提交
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
            op_.Type()));

    auto& out_var_names = op_.Outputs(out);

    for (size_t i = 0; i < in_var_list.size(); ++i) {
      if (out_var_names[i] == framework::kEmptyVarName) {
        continue;
      }

      Variable* in_var = in_var_list[i];
      if (!in_var->IsType<LoDTensor>()) return;
      Variable* out_var = out_var_list[i];
      PADDLE_ENFORCE_EQ(out_var->IsType<LoDTensor>(), true,
                        platform::errors::PreconditionNotMet(
                            "The %d-th output of Output(%s) must be LoDTensor.",
                            i, out_var_names[i]));
      auto& in_tensor = in_var->Get<LoDTensor>();
      auto* out_tensor = out_var->GetMutable<LoDTensor>();
      out_tensor->set_lod(in_tensor.lod());
#ifdef PADDLE_WITH_MKLDNN
      if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
        out_tensor->set_layout(in_tensor.layout());
    }
  }

Q
Qiao Longfei 已提交
797 798
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
799 800
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
    PADDLE_ENFORCE_NE(
        in_it, ctx_.inputs.end(),
        platform::errors::NotFound("Input %s does not exist.", in));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output %s does not exist.", out));
    PADDLE_ENFORCE_LT(i, in_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of input dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          in_it->second.size(), i));
    PADDLE_ENFORCE_LT(j, out_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of output dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          out_it->second.size(), j));
X
Xin Pan 已提交
817 818

    Variable* in_var = in_it->second.at(i);
Q
Qiao Longfei 已提交
819
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
820
    Variable* out_var = out_it->second.at(j);
821 822 823 824
    PADDLE_ENFORCE_EQ(
        out_var->IsType<LoDTensor>(), true,
        platform::errors::InvalidArgument(
            "The %zu-th output of Output(%s) must be LoDTensor.", j, out));
825
    auto& in_tensor = in_var->Get<LoDTensor>();
Q
Qiao Longfei 已提交
826 827
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
828

M
mozga-intel 已提交
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
848 849
  }

850
  int32_t GetLoDLevel(const std::string& in, size_t i = 0) const override {
851
    PADDLE_THROW(platform::errors::PreconditionNotMet(
852
        "GetLoDLevel is only used in compile time. The calculation of "
853
        "output's actual lod is different among operators so that should be "
854
        "set in the runtime kernel."));
855 856
  }

857 858
  void SetLoDLevel(const std::string& out, int32_t lod_level,
                   size_t j = 0) const override {
859
    PADDLE_THROW(platform::errors::PreconditionNotMet(
860
        "SetLoDLevel is only used in compile time. The calculation of "
861
        "output's actual lod is different among operators so that should be "
862
        "set in the runtime kernel."));
C
chengduo 已提交
863 864
  }

865 866
  bool IsRuntime() const override { return true; }

867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
  // TODO(paddle-dev): Can this be template?
  std::vector<InferShapeVarPtr> GetInputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = InputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

  std::vector<InferShapeVarPtr> GetOutputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = OutputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

X
Xin Pan 已提交
886 887
  DDim GetInputDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
888 889 890 891 892
    PADDLE_ENFORCE_EQ(
        vars.size(), 1UL,
        platform::errors::InvalidArgument(
            "Input(%s) should hold one element, but now it holds %zu elements.",
            name, vars.size()));
X
Xin Pan 已提交
893 894 895 896 897 898 899 900
    return this->GetDim(vars[0]);
  }

  std::vector<DDim> GetInputsDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    return GetDims(vars);
  }

X
Xin Pan 已提交
901 902 903 904 905 906 907 908 909 910
  std::vector<proto::VarType::Type> GetInputsVarType(
      const std::string& name) const override {
    return GetVarTypes(InputVars(name));
  }

  std::vector<proto::VarType::Type> GetOutputsVarType(
      const std::string& name) const override {
    return GetVarTypes(OutputVars(name));
  }

X
Xin Pan 已提交
911 912
  void SetOutputDim(const std::string& name, const DDim& dim) override {
    auto& vars = OutputVars(name);
913 914 915 916 917
    PADDLE_ENFORCE_EQ(
        vars.size(), 1UL,
        platform::errors::InvalidArgument("Output(%s) should hold one element, "
                                          "but now it holds %zu elements.",
                                          name, vars.size()));
X
Xin Pan 已提交
918 919 920 921 922 923 924 925 926
    SetDim(vars[0], dim);
  }

  void SetOutputsDim(const std::string& name,
                     const std::vector<DDim>& dims) override {
    auto& vars = OutputVars(name);
    SetDims(vars, dims);
  }

927
 protected:
X
Xin Pan 已提交
928
  DDim GetDim(Variable* var) const {
929 930
    PADDLE_ENFORCE_NOT_NULL(
        var, platform::errors::InvalidArgument("Input variable is nullptr."));
931 932 933 934 935
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
936 937 938 939
      PADDLE_THROW(platform::errors::InvalidArgument(
          "Only LoDTensor or SelectedRows support 'GetDim', but input "
          "Variable's type is %s.",
          ToTypeName(var->Type())));
F
fengjiayi 已提交
940 941 942
    }
  }

X
Xin Pan 已提交
943 944 945 946 947 948 949 950
  std::vector<DDim> GetDims(const std::vector<Variable*>& vars) const {
    std::vector<DDim> ret;
    ret.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(ret),
                   [this](Variable* var) { return this->GetDim(var); });
    return ret;
  }

F
fengjiayi 已提交
951
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
952 953
    PADDLE_THROW(platform::errors::PreconditionNotMet(
        "GetRepeatedDims method only ban be used in compile time."));
954 955
  }

X
Xin Pan 已提交
956
  void SetDim(Variable* var, const DDim& dim) {
957 958 959 960 961
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
962 963 964 965
      PADDLE_THROW(platform::errors::Unimplemented(
          "Variable type error, expect LoDTensor or SelectedRows, but received "
          "(%s).",
          ToTypeName(var->Type())));
X
Xin Pan 已提交
966 967 968 969 970 971
    }
  }

  void SetDims(const std::vector<Variable*>& vars,
               const std::vector<DDim>& dims) {
    size_t length = vars.size();
972 973 974 975 976 977
    PADDLE_ENFORCE_EQ(length, dims.size(),
                      platform::errors::InvalidArgument(
                          "The number of input variables do not match the "
                          "number of input dimensions, the number of variables "
                          "is %zu, the number of dimensions is %zu.",
                          length, dims.size()));
X
Xin Pan 已提交
978 979 980 981 982
    for (size_t i = 0; i < length; ++i) {
      if (vars[i] == nullptr) {
        continue;
      }
      SetDim(vars[i], dims[i]);
983 984 985
    }
  }

F
fengjiayi 已提交
986 987
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
988 989
    PADDLE_THROW(platform::errors::PreconditionNotMet(
        "SetRepeatedDims method only can be used in compile time."));
F
fengjiayi 已提交
990 991
  }

X
Xin Pan 已提交
992 993 994 995 996 997 998 999 1000 1001 1002
  std::vector<proto::VarType::Type> GetVarTypes(
      const std::vector<Variable*>& vars) const {
    std::vector<proto::VarType::Type> retv;
    retv.resize(vars.size());
    std::transform(vars.begin(), vars.end(), retv.begin(),
                   std::bind(std::mem_fn(&RuntimeInferShapeContext::GetVarType),
                             this, std::placeholders::_1));
    return retv;
  }

  proto::VarType::Type GetVarType(Variable* var) const {
1003 1004 1005
    return ToVarType(var->Type());
  }

1006 1007 1008
 private:
  const std::vector<Variable*>& InputVars(const std::string& name) const {
    auto it = ctx_.inputs.find(name);
1009 1010 1011 1012
    PADDLE_ENFORCE_NE(
        it, ctx_.inputs.end(),
        platform::errors::NotFound(
            "Operator (%s) does not have the input (%s).", op_.Type(), name));
1013 1014 1015 1016 1017
    return it->second;
  }

  const std::vector<Variable*>& OutputVars(const std::string& name) const {
    auto it = ctx_.outputs.find(name);
1018 1019 1020 1021
    PADDLE_ENFORCE_NE(
        it, ctx_.outputs.end(),
        platform::errors::NotFound(
            "Operator (%s) does not have the outputs (%s).", op_.Type(), name));
1022
    return it->second;
F
fengjiayi 已提交
1023 1024
  }

1025
  const OperatorBase& op_;
X
Xin Pan 已提交
1026
  const RuntimeContext& ctx_;
1027 1028
};

1029 1030
static void CheckTensorNANOrInf(const std::string& op_type,
                                const std::string& name,
C
chengduoZH 已提交
1031 1032 1033 1034
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
Y
Yu Yang 已提交
1035 1036
  if (tensor.type() != proto::VarType::FP32 &&
      tensor.type() != proto::VarType::FP64) {
C
chengduoZH 已提交
1037 1038
    return;
  }
1039 1040 1041 1042 1043 1044 1045 1046
  PADDLE_ENFORCE_NE(
      framework::TensorContainsInf(tensor), true,
      platform::errors::Fatal("Operator %s output Tensor %s contains Inf.",
                              op_type, name));
  PADDLE_ENFORCE_NE(
      framework::TensorContainsNAN(tensor), true,
      platform::errors::Fatal("Operator %s output Tensor %s contains NAN.",
                              op_type, name));
C
chengduoZH 已提交
1047 1048
}

1049 1050
bool OperatorWithKernel::SupportsMKLDNN(
    const proto::VarType::Type data_type) const {
1051 1052
  auto& op_kernels = OperatorWithKernel::AllOpKernels().at(type_);
  return std::any_of(op_kernels.begin(), op_kernels.end(),
1053
                     [data_type](OpKernelMap::const_reference kern_pair) {
1054 1055
                       return platform::is_cpu_place(kern_pair.first.place_) &&
                              kern_pair.first.library_type_ ==
1056 1057
                                  LibraryType::kMKLDNN &&
                              kern_pair.first.data_type_ == data_type;
1058 1059 1060
                     });
}

1061 1062
bool OperatorWithKernel::CanMKLDNNBeUsed(const framework::ExecutionContext& ctx,
                                         proto::VarType::Type data_type) const {
1063 1064
  bool use_mkldnn_ctx =
      ctx.Attr<bool>("use_mkldnn") && platform::is_cpu_place(ctx.GetPlace());
1065
  return use_mkldnn_ctx && this->SupportsMKLDNN(data_type);
1066 1067
}

B
baojun-nervana 已提交
1068
void OperatorWithKernel::RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
1069 1070
                                           const platform::Place& place,
                                           const RuntimeContext& ctx) const {
1071
  RuntimeInferShapeContext infer_shape_ctx(*this, ctx);
B
baojun-nervana 已提交
1072 1073 1074
  this->InferShape(&infer_shape_ctx);
}

L
luotao1 已提交
1075 1076
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
L
luotao1 已提交
1077 1078
  // To reduce the elapsed time of HasAttr, we use bool variable to record the
  // result of HasAttr.
1079 1080 1081
  if (!enable_cache_runtime_context_ && HasAttr(kEnableCacheRuntimeContext))
    enable_cache_runtime_context_ = true;
  if (!all_kernels_must_compute_runtime_shape_ &&
L
luotao1 已提交
1082
      HasAttr(kAllKernelsMustComputeRuntimeShape))
1083
    all_kernels_must_compute_runtime_shape_ = true;
1084
  const Scope* cur_scope = &scope;
1085
  if (!enable_cache_runtime_context_) {
L
luotao1 已提交
1086 1087
    RuntimeContext ctx(Inputs(), Outputs(), scope);
    RunImpl(scope, place, &ctx);
1088
    pre_scope_ = cur_scope;
L
luotao1 已提交
1089
  } else {
1090 1091 1092 1093 1094 1095
    if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
      std::lock_guard<std::mutex> lock(cache_update_mutex_);
      if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
        runtime_ctx_.reset(new RuntimeContext(Inputs(), Outputs(), scope));
        pre_scope_ = cur_scope;
      }
L
luotao1 已提交
1096 1097 1098 1099 1100 1101 1102 1103
    }
    RunImpl(scope, place, runtime_ctx_.get());
  }
}

void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place,
                                 RuntimeContext* runtime_ctx) const {
Y
Yu Yang 已提交
1104
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
1105
  auto* dev_ctx = pool.Get(place);
1106

1107
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
1108
    ChooseKernel(*runtime_ctx, scope, place);
1109 1110
  }

Y
yuyang18 已提交
1111 1112
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
1113 1114
  Scope* transfer_scope = nullptr;
  {
1115
    platform::RecordEvent record_event("prepare_data",
1116
                                       platform::EventRole::kInnerOp);
1117 1118 1119 1120
    if (need_prepare_data_) {
      transfer_scope = PrepareData(scope, *kernel_type_,
                                   &transfered_inplace_vars, runtime_ctx);
    }
1121
  }
Y
yuyang18 已提交
1122 1123 1124 1125
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

1126 1127
  if (!(kernel_type_->place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(kernel_type_->place_);
1128
  }
Q
QI JUN 已提交
1129

1130
  if (!all_kernels_must_compute_runtime_shape_) {
1131
    platform::RecordEvent record_event("infer_shape",
1132
                                       platform::EventRole::kInnerOp);
1133
    RuntimeInferShapeContext infer_shape_ctx(*this, *runtime_ctx);
1134 1135
    this->InferShape(&infer_shape_ctx);
  }
1136 1137 1138 1139 1140

  if (FLAGS_enable_unused_var_check) {
    GetThreadLocalUsedVarNameSet()->clear();
  }

X
clean  
Xin Pan 已提交
1141 1142
  // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
  // not Scope. Imperative mode only pass inputs and get outputs.
1143
  {
1144
    platform::RecordEvent record_event("compute",
1145
                                       platform::EventRole::kInnerOp);
1146 1147
    (*kernel_func_)(
        ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx));
1148
  }
D
dzhwinter 已提交
1149

Y
yuyang18 已提交
1150
  if (!transfered_inplace_vars.empty()) {
T
tianshuo78520a 已提交
1151
    // there is inplace variable has been transferred.
Y
yuyang18 已提交
1152
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
1153
  }
1154 1155 1156 1157 1158 1159 1160

  // See [ Why need handle complex gradient to real gradient? ]
  // Only handle the case where the current kernel data type is complex
  if (framework::IsComplexType(kernel_type_->data_type_)) {
    HandleComplexGradToRealGrad(scope, runtime_ctx);
  }

1161 1162 1163 1164 1165 1166 1167 1168
  if (FLAGS_enable_unused_var_check) {
    // skip op that uses mkldnn because it has different memory reuse strategy.
    // use attr here because some GradMakers (like ActivationGradOpMaker) add
    // input when use_mkldnn=true;
    if (!(HasAttr("use_mkldnn") && Attr<bool>("use_mkldnn"))) {
      CheckUnusedVar(*this, scope);
    }
  }
1169

D
dzhwinter 已提交
1170
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
1171
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
1172
    dev_ctx->Wait();
L
Leo Chen 已提交
1173 1174 1175
#if defined(PADDLE_WITH_CUDA)
    PADDLE_ENFORCE_CUDA_SUCCESS(cudaGetLastError());
    VLOG(4) << "Operator(" << Type() << "): context wait and get last error";
1176 1177 1178 1179
#endif
#if defined(PADDLE_WITH_HIP)
    PADDLE_ENFORCE_CUDA_SUCCESS(hipGetLastError());
    VLOG(4) << "Operator(" << Type() << "): context wait and get last error";
L
Leo Chen 已提交
1180
#endif
D
dzhwinter 已提交
1181
  }
C
chengduoZH 已提交
1182 1183

  if (FLAGS_check_nan_inf) {
W
WangXi 已提交
1184
    framework::details::CheckOpHasNanOrInf(*this, exec_scope, place);
C
chengduoZH 已提交
1185
  }
1186 1187 1188 1189 1190 1191 1192

  // To solve issue #15032, have a discussion with @Luotao for cpu inference,
  // do not cache transfer scope, hence in this case delete transfer scope
  // after run to avoid memory leak
  if (transfer_scope && !run_by_executor_ && !enable_cache_transfer_scope_) {
    scope.DeleteScope(transfer_scope);
  }
Q
Qiao Longfei 已提交
1193
}
X
Xin Pan 已提交
1194

L
Liu Yiqun 已提交
1195 1196 1197 1198 1199 1200 1201 1202 1203
void OperatorWithKernel::ChooseKernel(const RuntimeContext& ctx,
                                      const Scope& scope,
                                      const platform::Place& place) const {
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
  auto* dev_ctx = pool.Get(place);

  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
1204 1205 1206 1207 1208
  PADDLE_ENFORCE_NE(
      kernels_iter, all_op_kernels.end(),
      platform::errors::Unavailable(
          "There are no kernels which are registered in the %s operator.",
          type_));
L
Liu Yiqun 已提交
1209 1210 1211 1212

  OpKernelMap& kernels = kernels_iter->second;

  auto expected_kernel_key = this->GetExpectedKernelType(
1213
      ExecutionContext(*this, scope, *dev_ctx, ctx));
1214 1215 1216
  if (HasAttr("op_device")) {
    if (Attr<std::string>("op_device") == "cpu") {
      expected_kernel_key.place_ = platform::CPUPlace();
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
    } else if (Attr<std::string>("op_device").find("gpu") !=
               std::string::npos) {
      auto device = Attr<std::string>("op_device");
      size_t pos = device.find(':');
      if (pos != std::string::npos) {
        device = device.substr(0, pos);
        LOG_FIRST_N(WARNING, 1)
            << "Device index is only supported under pipeline parallelism, "
            << "so it will be ignored.";
      }
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
      // when the Op that only has CPUKernel is assigned to GPU, the CPUKernel
      // will be executed and a warning will be given at the same time.
      if (SupportGPU()) {
        expected_kernel_key.place_ = dev_ctx->GetPlace();
      } else {
        expected_kernel_key.place_ = platform::CPUPlace();
        LOG_FIRST_N(WARNING, 1)
            << "Op(" << type_
            << ") has no CUDA implementation. It will be assigned to CPUPlace.";
      }
    }
  }
L
Liu Yiqun 已提交
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
  VLOG(3) << "expected_kernel_key:" << expected_kernel_key;

  auto kernel_iter = kernels.find(expected_kernel_key);
#ifdef PADDLE_WITH_MKLDNN
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
    VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one";
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
#endif
#ifdef PADDLE_WITH_XPU
  if (kernel_iter == kernels.end() &&
      is_xpu_place(expected_kernel_key.place_)) {
    VLOG(3) << "missing XPU kernel: " << type_
            << ", expected_kernel_key:" << expected_kernel_key
            << ", fallbacking to CPU one!";
    expected_kernel_key.place_ = platform::CPUPlace();
    kernel_iter = kernels.find(expected_kernel_key);
  }
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
#endif
#ifdef PADDLE_WITH_ASCEND_CL
  if (kernel_iter == kernels.end() &&
      is_npu_place(expected_kernel_key.place_)) {
    VLOG(3) << "missing NPU kernel: " << type_
            << ", expected_kernel_key:" << expected_kernel_key
            << ", fallbacking to CPU one!";
    expected_kernel_key.place_ = platform::CPUPlace();
    kernel_iter = kernels.find(expected_kernel_key);
  }
L
Liu Yiqun 已提交
1271
#endif
1272 1273 1274 1275
  PADDLE_ENFORCE_NE(kernel_iter, kernels.end(),
                    platform::errors::NotFound(
                        "Operator (%s) does not have kernel for %s.", type_,
                        KernelTypeToString(expected_kernel_key)));
L
Liu Yiqun 已提交
1276

1277 1278 1279 1280 1281
  std::lock_guard<std::mutex> lock(cache_update_mutex_);
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
    kernel_type_.reset(new OpKernelType(expected_kernel_key));
    kernel_func_.reset(new OpKernelFunc(kernel_iter->second));
  }
L
Liu Yiqun 已提交
1282 1283
}

Y
yuyang18 已提交
1284 1285 1286 1287
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
M
minqiyang 已提交
1288
    VLOG(3) << "share inplace var " + var_name + " back to it's original scope";
C
chengduo 已提交
1289
    auto* origin_var = scope.FindVar(var_name);
1290 1291 1292
    PADDLE_ENFORCE_NOT_NULL(origin_var,
                            platform::errors::InvalidArgument(
                                "The variable[%s] is nullptr.", var_name));
C
chengduo 已提交
1293
    auto* original_tensor =
C
chengduo 已提交
1294
        GetMutableLoDTensorOrSelectedRowsValueFromVar(origin_var);
C
chengduo 已提交
1295
    auto* var = transfer_scope.FindVar(var_name);
1296 1297
    PADDLE_ENFORCE_NOT_NULL(var, platform::errors::InvalidArgument(
                                     "The variable[%s] is nullptr.", var_name));
C
chengduo 已提交
1298
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1299
    auto original_dims = original_tensor->dims();
Y
yuyang18 已提交
1300
    original_tensor->ShareDataWith(*transformed_tensor);
1301
    original_tensor->Resize(original_dims);
Y
yuyang18 已提交
1302 1303 1304
  }
}

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
void OperatorWithKernel::HandleComplexGradToRealGrad(
    const Scope& scope, RuntimeContext* ctx) const {
  for (auto& var_name_item : Outputs()) {
    std::vector<Variable*>& output_vars = ctx->outputs[var_name_item.first];
    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      // 1. find grad_var & check whether is complex tensor
      auto var_name = var_name_item.second[i];
      auto orig_var_name = GradOriginalVarName(var_name);
      // only focus on gradient var
      if (var_name == orig_var_name) {
        continue;
      }
      auto* grad_var = output_vars[i];
      // skip nullptr var
      if (grad_var == nullptr) {
        continue;
      }
      // don't process LoDTensorArray temporarily,
      // add support if necessary for complex number calculations in the future
      if (!VarIsTensor(*grad_var)) {
        continue;
      }
      auto* grad_tensor =
          GetMutableLoDTensorOrSelectedRowsValueFromVar(grad_var);
      // skip nullptr tensor
      if (grad_tensor == nullptr || !grad_tensor->IsInitialized()) {
        continue;
      }
      // only focus on complex dtype now
      auto src_type = grad_tensor->type();
      if (!IsComplexType(src_type)) {
        continue;
      }

      // 2. find forward var & check whether need to cast
      auto* var = scope.FindVar(orig_var_name);
      // if forward var not exists, do nothing
      if (var == nullptr) {
        continue;
      }
      if (!VarIsTensor(*var)) {
        continue;
      }
      const auto* tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
      PADDLE_ENFORCE_NOT_NULL(
          tensor,
          platform::errors::Unavailable(
              "Forward tensor is nullptr when handle complex data to real."));
      // only need record type, the allocation may have been released
      auto dst_type = tensor->saved_type();
      // only focus on real dtype and need casting
      if (IsComplexType(dst_type)) {
        continue;
      }

      // 3. cast complex grad to real grad
      VLOG(6) << "Transform " << framework::DataTypeToString(src_type)
              << " var `" << var_name << "` to "
              << framework::DataTypeToString(dst_type)
              << " real var in static graph.";
      Tensor out;
      TransComplexToReal(dst_type, src_type, *grad_tensor, &out);
      SetTensorToVariable(*grad_var, out, grad_var);
    }
  }
}

X
Xin Pan 已提交
1372
Scope* OperatorWithKernel::PrepareData(
Y
yuyang18 已提交
1373
    const Scope& scope, const OpKernelType& expected_kernel_key,
X
Xin Pan 已提交
1374 1375
    std::vector<std::string>* transfered_inplace_vars,
    RuntimeContext* ctx) const {
Y
yuyang18 已提交
1376
  Scope* new_scope = nullptr;
S
sneaxiy 已提交
1377

1378
  const std::unordered_set<std::string>* no_buffer_ins = nullptr;
S
sneaxiy 已提交
1379 1380 1381 1382
  if (info_) {
    auto& no_buffer_inferer = info_->NoNeedBufferVarsInferer();
    // Some op may not register NoNeedBufferVarsInferer
    if (no_buffer_inferer) {
1383 1384
      no_buffer_ins = &(no_buffer_inferer(Inputs(), Outputs(), Attrs()));
      if (no_buffer_ins->empty()) no_buffer_ins = nullptr;
S
sneaxiy 已提交
1385 1386 1387
    }
  }

Y
yuyang18 已提交
1388
  for (auto& var_name_item : Inputs()) {
1389 1390
    bool should_skip_input =
        no_buffer_ins && no_buffer_ins->count(var_name_item.first) > 0;
S
sneaxiy 已提交
1391

X
Xin Pan 已提交
1392 1393 1394 1395
    std::vector<Variable*>& input_vars = ctx->inputs[var_name_item.first];

    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      auto& var_name = var_name_item.second[i];
X
Xin Pan 已提交
1396
      auto* var = input_vars[i];
X
Xin Pan 已提交
1397

Y
yuyang18 已提交
1398
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
1399
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
1400 1401 1402
        continue;
      }

C
chengduo 已提交
1403
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418

      // When no_buffer_ins then checking of Tensor::holder_ is
      // not a thread safe. And for infershape scenario checks
      // to be omitted are not really needed
      if (should_skip_input == true) {
#ifdef PADDLE_WITH_MKLDNN
        // Var without buffer may be needed
        // for some situation like InferShape().
        // In this situation We cannot skip Var analysis, as
        // MKL-DNN shape of Var may differ from kNHWC Var
        // In such situation corressponding resized Var
        // has to be created and registered
        if ((tensor_in->layout() == DataLayout::kMKLDNN) &&
            (var->IsType<LoDTensor>() == true) &&
            (expected_kernel_key.data_layout_ != DataLayout::kMKLDNN) &&
1419 1420
            (paddle::platform::MKLDNNDeviceContext::tls()
                 .get_cur_paddle_data_layout() == DataLayout::kNHWC)) {
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
          // Mixed execution : MKL-DNN and GPU is not supported!
          if (!new_scope) {
            new_scope = &scope.NewScope();
          }
          auto* trans_var = new_scope->Var(var_name);
          input_vars[i] = trans_var;
          auto out = trans_var->GetMutable<LoDTensor>();
          out->Resize(tensor_in->dims());
          platform::MatchShapeToLayout(out, tensor_in->layout(),
                                       DataLayout::kNHWC);
          VLOG(7) << "Created reshaped dummy input based on MKL-DNN Tensor , "
                     "but kNHWC layout"
                  << var_name_item.first << " in Operator " << type_;
        } else {
          VLOG(7) << "Skip scanning input " << var_name_item.first
                  << " in Operator " << type_;
        }
#endif
        continue;
      }

Y
yuyang18 已提交
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

M
minqiyang 已提交
1453 1454
      VLOG(3) << "Transform Variable " << var_name << " from "
              << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
1455

1456 1457 1458
      // In the inference scenerio, the scopes will be reused across the
      // batches, so the `new_scope` here will result in GPU memroy explosion
      // over the  running of operators.
1459
      // We use a thread_local cache to fix that issue, the key in the cache is
1460 1461 1462 1463 1464
      // the combination of the `scope` argument, from_kernel_type,
      // target_kernel_type.
      // Have a discussion with @Superjomn or the inference developers if some
      // changes on this logic for this macro might not tested on the other
      // scenerios.
1465 1466
      // If this op is not called by an Executor or ParallelExecutor, it should
      // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and
1467
      // variables, that behavior a lot different.
1468 1469 1470 1471 1472 1473 1474 1475 1476
      //
      // To solve issue #15032, have a discussion with @Luotao for cpu
      // inference, for all cpu kernels cases without GPU participation, here
      // not do transfer scope caching, and cpu inference performance is not
      // impacted by test.
      enable_cache_transfer_scope_ = false;
      if (!run_by_executor_ &&
          (platform::is_gpu_place(kernel_type_for_var.place_) ||
           platform::is_gpu_place(expected_kernel_key.place_))) {
1477 1478
        new_scope = TryCreateTransferScope(kernel_type_for_var,
                                           expected_kernel_key, &scope);
1479
        enable_cache_transfer_scope_ = true;
1480
      }
1481
      if (!new_scope) {
Y
yuyang18 已提交
1482 1483
        new_scope = &scope.NewScope();
      }
1484 1485 1486 1487
      // For inference, if a gpu model has an op which could only run on CPU,
      // each result of different input will be the same with the first one.
      // The reason is that if a gpu tensor is the input of a cpu kernel,
      // we will create a new cpu tensor in new scope.
1488
      // However, if enable_cache_runtime_context_, we get the cpu tensor each
1489 1490
      // time, not the gpu tensor. Thus, we set pre_scope_ = nullptr
      // to trigger `new RuntimeContext()` in RunImpl().
1491
      if (enable_cache_runtime_context_) {
1492 1493
        pre_scope_ = nullptr;
      }
L
Leo Chen 已提交
1494 1495

      // Create new var with the same name in transfer scopes
Y
yuyang18 已提交
1496
      auto* trans_var = new_scope->Var(var_name);
X
fix  
Xin Pan 已提交
1497
      input_vars[i] = trans_var;
L
Leo Chen 已提交
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514

      // Find if inplace exists between input and output
      // If inplace exists, set the new created var to inplaced output, and
      // record its name in transfered_inplace_vars.
      for (auto& pair : Outputs()) {
        for (size_t j = 0; j < pair.second.size(); ++j) {
          if (pair.second[j] == var_name) {
            VLOG(4) << "Found inplace between input(" << var_name_item.first
                    << ") and output(" << pair.first
                    << "), the variable name is " << var_name;
            ctx->outputs[pair.first][j] = trans_var;
            transfered_inplace_vars->emplace_back(var_name);
          }
        }
      }

      // Do transfer
Y
yuyang18 已提交
1515
      Tensor out;
Y
yuyang18 已提交
1516
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
1517 1518 1519
      SetTensorToVariable(*var, out, trans_var);
    }
  }
L
Leo Chen 已提交
1520

1521 1522 1523 1524 1525 1526 1527 1528 1529
  // If pre_scope = &scope, it means that scope is cached and the op is not in
  // while block. If new_scope = nullptr, it means that for each input of this
  // Op, there is no need to do PrepareData. So PrepareData could be skipped at
  // the rest iterations to save the elapsed time.
  // We do not support skipping PrepareData in while block, because the Op's
  // input may be changed by subsequent Ops, which may cause an error.
  if (pre_scope_ == &scope && new_scope == nullptr) {
    need_prepare_data_ = false;
  }
Y
yuyang18 已提交
1530 1531 1532

  return new_scope;
}
Q
Qiao Longfei 已提交
1533

1534 1535 1536
void OperatorWithKernel::ParseInputDataType(
    const ExecutionContext& ctx, const std::string& name,
    proto::VarType::Type* data_type) const {
1537
  proto::VarType::Type default_data_type =
1538
      static_cast<proto::VarType::Type>(-1);
H
hong 已提交
1539
  const std::vector<Variable*> vars = ctx.MultiInputVar(name);
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
  for (size_t i = 0; i < vars.size(); ++i) {
    const Variable* var = vars[i];
    if (var != nullptr) {
      const Tensor* t = nullptr;
      if (var->IsType<Tensor>()) {
        t = &var->Get<Tensor>();
      } else if (var->IsType<LoDTensor>()) {
        t = &var->Get<LoDTensor>();
      } else if (var->IsType<SelectedRows>()) {
        t = &(var->Get<SelectedRows>().value());
1550 1551 1552 1553 1554 1555 1556
      } else if (var->IsType<LoDTensorArray>()) {
        auto t_arr = var->Get<LoDTensorArray>();
        for (size_t j = 0; j < t_arr.size(); j++) {
          if (t_arr[j].IsInitialized()) {
            t = &(t_arr[j]);
          }
        }
1557 1558
      }
      if (t != nullptr) {
1559 1560 1561 1562 1563
        PADDLE_ENFORCE_EQ(
            t->IsInitialized(), true,
            platform::errors::InvalidArgument(
                "The Tensor in the %s Op's Input Variable %s(%s) is "
                "not initialized.",
H
hong 已提交
1564
                Type(), name, ctx.InputNames(name).at(i)));
1565
        proto::VarType::Type tmp = t->type();
1566
        PADDLE_ENFORCE(
1567
            tmp == *data_type || *data_type == default_data_type,
1568 1569 1570 1571 1572 1573
            platform::errors::InvalidArgument(
                "The DataType of %s Op's duplicable Variable %s must be "
                "consistent. The current variable type is (%s), but the "
                "previous variable type is (%s).",
                Type(), name, DataTypeToString(tmp),
                DataTypeToString(*data_type)));
1574 1575 1576 1577 1578 1579
        *data_type = tmp;
      }
    }
  }
}

1580
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
1581
    const ExecutionContext& ctx) const {
1582 1583 1584
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
H
hong 已提交
1585 1586
  for (auto& input : ctx.InNameList()) {
    ParseInputDataType(ctx, input, &data_type);
Y
Yu Yang 已提交
1587
  }
1588 1589 1590 1591
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
      platform::errors::NotFound(
          "DataType should be indicated by input Variable at %s.", Type()));
1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
  return data_type;
}

proto::VarType::Type OperatorWithKernel::IndicateVarDataType(
    const ExecutionContext& ctx, const std::string& name) const {
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
  ParseInputDataType(ctx, name, &data_type);
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
1603 1604 1605 1606 1607
      platform::errors::InvalidArgument(
          "The Input Variable(%s) of (%s) Operator used to determine kernel "
          "data type is empty or not LoDTensor or SelectedRows or "
          "LoDTensorArray.",
          name, Type()));
1608
  return data_type;
Y
Yu Yang 已提交
1609
}
1610

1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
Tensor* OperatorWithKernel::GetTensorFormInputSafely(
    const ExecutionContext& ctx, const std::string& name) const {
  // 1. get variable and check
  // NOTE: only supports signal input var now
  // NOTE: using const_cast is because we don't have method
  // can get single mutable var, and here will not change
  // the var's data, only use some attribute
  Variable* var = const_cast<Variable*>(ctx.InputVar(name));
  PADDLE_ENFORCE_NOT_NULL(
      var,
      platform::errors::NotFound(
          "The variable %s is not found when promote complex types.", name));
  // 2. get tensor and check
  Tensor* t = nullptr;
  if (var->IsType<Tensor>()) {
    t = var->GetMutable<Tensor>();
  } else if (var->IsType<LoDTensor>()) {
    t = var->GetMutable<LoDTensor>();
  } else if (var->IsType<SelectedRows>()) {
    t = var->GetMutable<SelectedRows>()->mutable_value();
  } else {
    PADDLE_THROW(platform::errors::Unimplemented(
        "Unsupported input variable type in complex type promotion."));
  }
  PADDLE_ENFORCE_NOT_NULL(
      t,
      platform::errors::InvalidArgument(
          "The Tensor of variable %s is nullptr when promote complex types."));
  PADDLE_ENFORCE_EQ(t->IsInitialized(), true,
                    platform::errors::InvalidArgument(
                        "The Tensor in the %s Op's Input Variable %s(%s) is "
                        "not initialized.",
                        Type(), name, ctx.InputName(name)));
  return t;
}

/** NOTE(chenweihang): For safety reasons, we now only
 * perform type promotes for binary operations with
 * complex type inputs, which is used to support the
 * paddle quantum function.
 * In other cases, the first input data type is used as
 * the kernel data type.
 */
proto::VarType::Type OperatorWithKernel::IndicateOrPromoteVarDataTypes(
    const ExecutionContext& ctx, const std::string& name1,
    const std::string& name2) const {
  // 1. Get tensor
  auto* tensor_a = GetTensorFormInputSafely(ctx, name1);
  auto* tensor_b = GetTensorFormInputSafely(ctx, name2);

  // 2. Get two input types
  auto type_a = tensor_a->type();
  auto type_b = tensor_b->type();

  // 3. Get first input type or promote complex types
  auto target_type = PromoteTypesIfComplexExists(type_a, type_b);

  return target_type;
}

1671 1672 1673 1674 1675 1676 1677 1678
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
1679 1680
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
1681 1682
}

Q
Qiao Longfei 已提交
1683
}  // namespace framework
L
liaogang 已提交
1684
}  // namespace paddle