operator.cc 73.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include "paddle/fluid/framework/operator.h"

17
#include <glog/logging.h>
P
peizhilin 已提交
18 19
#include <sstream>
#include <string>
20

21
#include "gflags/gflags.h"
Y
Yi Wang 已提交
22
#include "paddle/fluid/framework/data_transform.h"
23
#include "paddle/fluid/framework/data_type_transform.h"
W
WangXi 已提交
24
#include "paddle/fluid/framework/details/nan_inf_utils.h"
25
#include "paddle/fluid/framework/op_call_stack.h"
Y
Yi Wang 已提交
26
#include "paddle/fluid/framework/shape_inference.h"
27
#include "paddle/fluid/framework/transfer_scope_cache.h"
28
#include "paddle/fluid/framework/unused_var_check.h"
Y
Yi Wang 已提交
29
#include "paddle/fluid/framework/var_type.h"
L
Leo Chen 已提交
30
#include "paddle/fluid/platform/enforce.h"
31
#include "paddle/fluid/platform/profiler.h"
32
#include "paddle/pten/common/scalar.h"
33 34 35 36 37 38

namespace paddle {
namespace framework {
class LoDTensor;
}  // namespace framework
}  // namespace paddle
39
#ifdef PADDLE_WITH_XPU
Q
QingshuChen 已提交
40 41
#include "paddle/fluid/platform/xpu/xpu_info.h"
#include "paddle/fluid/platform/xpu/xpu_op_list.h"
42
#endif
Q
Qiao Longfei 已提交
43

44 45 46 47
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/platform/mkldnn_helper.h"
#endif

D
dzhwinter 已提交
48
DECLARE_bool(benchmark);
49
DECLARE_bool(check_nan_inf);
50
DECLARE_bool(enable_unused_var_check);
51 52
PADDLE_DEFINE_EXPORTED_int32(inner_op_parallelism, 0,
                             "number of threads for inner op");
53
DECLARE_bool(run_pten_kernel);
D
dzhwinter 已提交
54

Q
Qiao Longfei 已提交
55 56 57
namespace paddle {
namespace framework {

58 59 60 61 62 63
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
64

65
static DDim GetDimsDebug(const ScopeBase& scope, const std::string& name,
66
                         bool get_actual_dim = false) {
67
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
68 69
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
70 71
  }

M
minqiyang 已提交
72 73 74 75 76 77 78 79 80
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
S
Steffy-zxf 已提交
81 82
  } else if (var->IsType<Strings>()) {
    return DDim({static_cast<int64_t>(var->Get<Strings>().size())});
83 84 85 86 87
  } else {
    return DDim({-1});
  }
}

88
static bool VarInited(const ScopeBase& scope, const std::string& name) {
Q
Qiao Longfei 已提交
89 90 91 92 93
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

94
static std::string GetDtype(const ScopeBase& scope, const std::string& name) {
D
dzhwinter 已提交
95 96 97 98
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
99

M
minqiyang 已提交
100 101 102
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
103 104
      return "";
    }
Y
Yu Yang 已提交
105
    return DataTypeToString(tensor.type());
M
minqiyang 已提交
106
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
107 108 109 110
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
Y
Yu Yang 已提交
111
      return DataTypeToString(tensor.type());
Q
Qiao Longfei 已提交
112
    }
S
Steffy-zxf 已提交
113 114
  } else if (var->IsType<Strings>()) {
    return "strings";
D
dzhwinter 已提交
115 116 117 118 119
  } else {
    return "";
  }
}

120
static std::string GetPlace(const ScopeBase& scope, const std::string& name) {
L
Leo Chen 已提交
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
  auto to_string = [](const platform::Place& p) {
    std::stringstream sstream;
    sstream << p;
    return sstream.str();
  };

  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "";
    }
    return to_string(tensor.place());
  } else if (var->IsType<SelectedRows>()) {
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
      return to_string(tensor.place());
    }
  } else {
    return "";
  }
}

149
static int GetRowSize(const ScopeBase& scope, const std::string& name) {
150 151 152 153 154
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
155 156
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
157 158 159 160 161
  }

  return -1;
}

162
static LoD GetLoDDebug(const ScopeBase& scope, const std::string& name) {
Q
Qiao Longfei 已提交
163 164 165 166 167 168 169
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
170 171 172
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.lod();
Q
Qiao Longfei 已提交
173 174 175 176 177
  } else {
    return default_lod;
  }
}

X
Xin Pan 已提交
178 179 180 181 182
RuntimeContext::RuntimeContext(const VariableNameMap& innames,
                               const VariableNameMap& outnames,
                               const Scope& scope) {
  for (auto& var_name_item : innames) {
    std::vector<Variable*>& input_vars = inputs[var_name_item.first];
X
Xin Pan 已提交
183
    input_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
184 185 186 187 188 189
    for (auto& var_name : var_name_item.second) {
      input_vars.push_back(scope.FindVar(var_name));
    }
  }
  for (auto& var_name_item : outnames) {
    std::vector<Variable*>& output_vars = outputs[var_name_item.first];
X
Xin Pan 已提交
190
    output_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
191 192 193 194 195 196
    for (auto& var_name : var_name_item.second) {
      output_vars.push_back(scope.FindVar(var_name));
    }
  }
}

197
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
P
peizhilin 已提交
198 199 200
  try {
    VLOG(4) << place << " " << DebugStringEx(&scope);
    if (platform::is_gpu_place(place)) {
201
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
202 203 204 205
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with CUDA support.",
          place));
206
#else
207
      auto dev_id = BOOST_GET_CONST(platform::CUDAPlace, place).device;
P
peizhilin 已提交
208
      platform::SetDeviceId(dev_id);
209 210 211
#endif
    } else if (platform::is_xpu_place(place)) {
#ifndef PADDLE_WITH_XPU
212 213 214 215
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with XPU support.",
          place));
216 217 218
#else
      auto dev_id = BOOST_GET_CONST(platform::XPUPlace, place).device;
      platform::SetXPUDeviceId(dev_id);
219 220 221 222 223 224 225 226 227 228
#endif
    } else if (platform::is_npu_place(place)) {
#ifndef PADDLE_WITH_ASCEND_CL
      PADDLE_THROW(platform::errors::Unavailable(
          "Cannot run operator on place %s, please recompile paddle or "
          "reinstall Paddle with NPU support.",
          place));
#else
      auto dev_id = BOOST_GET_CONST(platform::NPUPlace, place).device;
      platform::SetNPUDeviceId(dev_id);
229
#endif
P
peizhilin 已提交
230
    }
P
peizhilin 已提交
231

232
    {
233 234 235 236 237 238
      // TODO(wangchaochaohu) : refine code to use only one RecordEvent)
      // in order to record different op type cost time
      // and different op name cost time,we set two event.
      platform::RecordEvent op_type_record_event(Type());
      auto op_name = platform::OpName(outputs_, Type());
      platform::RecordEvent op_name_record_event(
239
          op_name, platform::EventRole::kUniqueOp);
P
peizhilin 已提交
240 241
      RunImpl(scope, place);
    }
242

Z
Zhang Ting 已提交
243
    VLOG(3) << GetExecutionPlace(place) << " " << DebugStringEx(&scope);
244
  } catch (platform::EnforceNotMet& exception) {
245
    framework::InsertCallStackInfo(Type(), Attrs(), &exception);
246
    throw std::move(exception);
247 248 249 250 251 252
  } catch (platform::EOFException&) {
    std::rethrow_exception(std::current_exception());
  } catch (std::exception& ex) {
    LOG(WARNING) << Type() << " raises an exception "
                 << platform::demangle(typeid(ex).name()) << ", " << ex.what();
    std::rethrow_exception(std::current_exception());
P
peizhilin 已提交
253
  } catch (...) {
254
    LOG(WARNING) << Type() << " raises an unknown exception";
P
peizhilin 已提交
255
    std::rethrow_exception(std::current_exception());
256
  }
257 258
}

259
bool OperatorBase::HasInputs(const std::string& name) const {
M
minqiyang 已提交
260
  return inputs_.find(name) != inputs_.end();
261 262
}

263
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
264
  auto& ins = Inputs(name);
265 266
  PADDLE_ENFORCE_LE(
      ins.size(), 1UL,
267
      platform::errors::InvalidArgument(
268 269
          "Operator %s's input %s should contain only one variable.", type_,
          name));
Y
Yu Yang 已提交
270
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
271 272
}

Y
Yu Yang 已提交
273 274
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
275
  auto it = inputs_.find(name);
276 277 278 279
  PADDLE_ENFORCE_NE(
      it, inputs_.end(),
      platform::errors::NotFound("Operator %s does not have the input %s.",
                                 type_, name));
Y
Yu Yang 已提交
280
  return it->second;
Y
Yan Chunwei 已提交
281 282
}

283
bool OperatorBase::HasOutputs(const std::string& name) const {
284
  if (outputs_.find(name) != outputs_.end()) {
285 286 287 288 289 290
    return true;
  } else {
    return false;
  }
}

291
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
292
  auto& outs = Outputs(name);
293 294 295 296 297
  PADDLE_ENFORCE_LE(
      outs.size(), 1UL,
      platform::errors::InvalidArgument(
          "Operator %s's output %s should contain only one variable.", type_,
          name));
Y
Yu Yang 已提交
298
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
299 300
}

Y
Yu Yang 已提交
301 302
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
303
  auto it = outputs_.find(name);
304 305 306 307
  PADDLE_ENFORCE_NE(
      it, outputs_.end(),
      platform::errors::NotFound(
          "Operator %s does not have an output called %s.", type_, name));
Y
Yu Yang 已提交
308
  return it->second;
Y
Yan Chunwei 已提交
309 310
}

311
std::string OperatorBase::DebugStringEx(const ScopeBase* scope) const {
Q
Qiao Longfei 已提交
312
  std::stringstream ss;
Y
Yu Yang 已提交
313
  ss << "Op(" << type_ << "), inputs:{";
314

315
  const std::unordered_set<std::string>* no_need_buffer_vars = nullptr;
316 317
  if (info_ && info_->NoNeedBufferVarsInferer()) {
    no_need_buffer_vars =
318 319
        &(Info().NoNeedBufferVarsInferer()(Inputs(), Outputs(), Attrs()));
    if (no_need_buffer_vars->empty()) no_need_buffer_vars = nullptr;
320 321
  }

Y
Yu Yang 已提交
322 323
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
324 325
    bool is_no_need_buffer_var =
        (no_need_buffer_vars && no_need_buffer_vars->count(input.first) > 0);
Y
Yu Yang 已提交
326 327
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
328 329
      auto var_name = input.second[i];
      ss << var_name;
330
      if (scope) {
Q
Qiao Longfei 已提交
331 332 333 334 335 336 337
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
338 339 340
          std::string dtype = is_no_need_buffer_var
                                  ? "unknown_dtype"
                                  : GetDtype(*scope, var_name);
Q
Qiao Longfei 已提交
341
          ss << ":" << dtype;
342 343
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
L
Leo Chen 已提交
344
          ss << "(" << GetPlace(*scope, var_name) << ")";
345
        }
346
      }
Y
Yu Yang 已提交
347 348 349
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
350
    }
Y
Yu Yang 已提交
351
    ss << "]";
Y
Yu Yang 已提交
352 353
    ++it;
    if (it != inputs_.end()) {
354 355
      ss << ", ";
    }
Q
Qiao Longfei 已提交
356
  }
Y
Yu Yang 已提交
357
  ss << "}, outputs:{";
Y
Yu Yang 已提交
358 359
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
360 361
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
362 363
      auto var_name = output.second[i];
      ss << var_name;
364
      if (scope) {
Q
Qiao Longfei 已提交
365 366 367 368 369 370 371
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
372 373
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
374 375
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
L
Leo Chen 已提交
376
          ss << "(" << GetPlace(*scope, var_name) << ")";
377
        }
378
      }
Y
Yu Yang 已提交
379 380 381
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
382
    }
Y
Yu Yang 已提交
383
    ss << "]";
Y
Yu Yang 已提交
384 385
    ++it;
    if (it != outputs_.end()) {
386 387
      ss << ", ";
    }
Q
Qiao Longfei 已提交
388
  }
Y
Yu Yang 已提交
389
  ss << "}.";
Q
Qiao Longfei 已提交
390 391 392
  return ss.str();
}

Y
Yu Yang 已提交
393
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
394 395
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
396
                           const AttributeMap& attrs)
S
sneaxiy 已提交
397 398 399 400 401 402
    : type_(type),
      inputs_(inputs),
      outputs_(outputs),
      attrs_(attrs),
      // NOTE(zjl): why op_info may be nullptr?
      info_(OpInfoMap::Instance().GetNullable(type)) {
H
hong 已提交
403 404 405 406 407 408 409 410
  // In dygraph mode, all the OperatorBase will be constructed by function:
  // framework::OpRegistry::CreateOp(type, {}, {}, {}, false).
  // Inputs, outputs and attrs will be set to empty map
  // to improve the execution efficiency of dygraph.
  if (inputs_.size() > 0 || outputs_.size() > 0) {
    GenerateTemporaryNames();
    CheckAllInputOutputSet();
  }
Y
Yu Yang 已提交
411
}
412

Q
qijun 已提交
413 414
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
415
  for (auto& o : inputs_) {
Q
qijun 已提交
416 417 418 419 420 421
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
422 423 424 425 426 427 428 429 430 431
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
S
sneaxiy 已提交
432
  auto& info = Info();
Y
Yu Yang 已提交
433 434

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
435
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
436 437 438 439 440 441 442 443 444
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
445 446
}

447
void OperatorBase::CheckAllInputOutputSet() const {
S
sneaxiy 已提交
448
  if (info_ == nullptr || info_->proto_ == nullptr) return;
449

S
sneaxiy 已提交
450
  for (auto& in : info_->Proto().inputs()) {
451
    if (!in.dispensable() && !in.extra()) {
452 453 454 455
      PADDLE_ENFORCE_NE(
          inputs_.find(in.name()), inputs_.end(),
          platform::errors::NotFound("Operator %s's input (%s) is not set.",
                                     Type(), in.name()));
456
    }
457 458
  }

S
sneaxiy 已提交
459
  for (auto& out : info_->Proto().outputs()) {
460
    if (!out.dispensable() && !out.extra()) {
461 462 463 464
      PADDLE_ENFORCE_NE(
          outputs_.find(out.name()), outputs_.end(),
          platform::errors::NotFound("Operator %s's output (%s) is not set.",
                                     Type(), out.name()));
465
    }
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

B
baojun-nervana 已提交
482
static bool VarIsTensor(const Variable& var) {
C
chengduo 已提交
483
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
484 485
}

C
chengduo 已提交
486
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
487 488 489 490
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
491
  } else {
492 493 494
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Variable type is %s, expect LoDTensor or SelectedRows.",
        ToTypeName(var.Type())));
Q
QI JUN 已提交
495 496 497
  }
}

C
chengduo 已提交
498
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
499
  if (var->IsType<LoDTensor>()) {
500
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
501
  } else if (var->IsType<SelectedRows>()) {
502
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
503
  } else {
504 505 506
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Variable type is %s, expect LoDTensor or SelectedRows.",
        ToTypeName(var->Type())));
Q
QI JUN 已提交
507 508 509
  }
}

510
bool ExecutionContext::HasInput(const std::string& name) const {
511
  auto* var = InputVar(name);
512 513 514 515
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
516
  auto* var = OutputVar(name);
517 518 519
  return var != nullptr;
}

X
Xin Pan 已提交
520
const Variable* ExecutionContext::InputVar(const std::string& name) const {
521 522
  LogVarUsageIfUnusedVarCheckEnabled(name);

X
Xin Pan 已提交
523 524 525
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) return nullptr;

526 527
  PADDLE_ENFORCE_LE(
      it->second.size(), 1UL,
528
      platform::errors::InvalidArgument(
529 530
          "Operator %s's input %s should contain only one variable.",
          op_.Type(), name));
X
Xin Pan 已提交
531 532 533
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
534
Variable* ExecutionContext::OutputVar(const std::string& name) const {
X
Xin Pan 已提交
535 536 537
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) return nullptr;

538 539 540 541 542
  PADDLE_ENFORCE_LE(
      it->second.size(), 1UL,
      platform::errors::InvalidArgument(
          "Operator %s's output %s should contain only one variable.",
          op_.Type(), name));
X
Xin Pan 已提交
543 544 545
  return it->second.empty() ? nullptr : it->second[0];
}

546
template <>
547
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
548
  return Input<LoDTensor>(name);
549 550 551
}

template <>
552
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
553
    const std::string& name) const {
554 555
  LogVarUsageIfUnusedVarCheckEnabled(name);

H
hong 已提交
556 557
  auto vars = MultiInputVar(name);
  if (vars.size() == 0) {
X
Xin Pan 已提交
558 559 560 561 562
    return {};
  }
  std::vector<const Tensor*> res;
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
H
hong 已提交
563
                 [&](const Variable* var) -> const Tensor* {
X
Xin Pan 已提交
564
                   if (var == nullptr) return nullptr;
565 566 567 568 569
                   PADDLE_ENFORCE_EQ(var->IsType<LoDTensor>(), true,
                                     platform::errors::InvalidArgument(
                                         "Input variable should be LoDTensor, "
                                         "but the received type is %s.",
                                         ToTypeName(var->Type())));
X
Xin Pan 已提交
570 571 572 573 574
                   return &(var->Get<LoDTensor>());
                 });
  return res;
}

575
template <>
576
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
577
  return Output<LoDTensor>(name);
578 579 580
}

template <>
581
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
582
    const std::string& name) const {
H
hong 已提交
583 584 585
  auto vars = MultiOutputVar(name);

  if (vars.size() == 0) {
586 587
    return {};
  }
588
  std::vector<Tensor*> res;
589 590 591 592 593
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                 [&](Variable* var) -> Tensor* {
                   return var == nullptr ? nullptr
                                         : var->GetMutable<LoDTensor>();
594
                 });
595 596 597
  return res;
}

Y
Yu Yang 已提交
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

613 614
class RuntimeInferShapeContext : public InferShapeContext {
 public:
615
  RuntimeInferShapeContext(const OperatorBase& op, const RuntimeContext& ctx)
G
Gabor Buella 已提交
616
      : op_(op), ctx_(ctx) {}
617 618

  bool HasInput(const std::string& name) const override {
619
    // has only one input
X
Xin Pan 已提交
620
    const auto& ins = ctx_.inputs;
621 622
    auto it = ins.find(name);
    if (it == ins.end()) {
623 624
      return false;
    }
625
    const auto& in = it->second;
X
Xin Pan 已提交
626
    if (in.size() == 0) return false;
627 628 629 630
    PADDLE_ENFORCE_EQ(
        in.size(), 1UL,
        platform::errors::InvalidArgument(
            "Input %s should not contain more than one inputs.", name));
X
Xin Pan 已提交
631
    return in[0] != nullptr;
632 633 634
  }

  bool HasOutput(const std::string& name) const override {
635
    // has only one output
X
Xin Pan 已提交
636
    const auto& outs = ctx_.outputs;
637 638
    auto it = outs.find(name);
    if (it == outs.end()) {
639 640
      return false;
    }
641
    const auto& out = it->second;
X
Xin Pan 已提交
642
    if (out.size() == 0) {
643 644
      return false;
    }
645 646 647 648
    PADDLE_ENFORCE_EQ(
        out.size(), 1UL,
        platform::errors::InvalidArgument(
            "Output %s should not contain more than one outputs.", name));
X
Xin Pan 已提交
649
    return out[0] != nullptr;
650 651 652
  }

  bool HasInputs(const std::string& name) const override {
X
Xin Pan 已提交
653 654
    const auto& ins = ctx_.inputs;
    auto it = ins.find(name);
X
fix  
Xin Pan 已提交
655
    if (it == ins.end() || it->second.empty()) {
656 657
      return false;
    }
X
Xin Pan 已提交
658 659
    for (auto& input : it->second) {
      if (input == nullptr) {
660 661 662 663 664 665 666
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
X
Xin Pan 已提交
667 668
    const auto& outs = ctx_.outputs;
    auto it = outs.find(name);
X
fix  
Xin Pan 已提交
669
    if (it == outs.end() || it->second.empty()) {
670 671
      return false;
    }
X
Xin Pan 已提交
672 673
    for (auto& output : it->second) {
      if (output == nullptr) {
674 675 676 677 678 679 680 681
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

H
hong 已提交
682
  std::vector<std::string> Inputs(const std::string& name) const override {
683 684 685
    return op_.Inputs(name);
  }

H
hong 已提交
686
  std::vector<std::string> Outputs(const std::string& name) const override {
687 688 689
    return op_.Outputs(name);
  }

690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
  std::string GetInputNameByIdx(size_t idx) const override {
    auto& op_proto =
        paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_;
    PADDLE_ENFORCE_LT(idx, op_proto->inputs().size(),
                      platform::errors::OutOfRange(
                          "The index should be less than the size of inputs of "
                          "operator %s, but got index is %d and size is %d",
                          op_.Type(), idx, op_proto->inputs().size()));
    return op_proto->inputs()[idx].name();
  }

  std::string GetOutputNameByIdx(size_t idx) const override {
    auto& op_proto =
        paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_;
    PADDLE_ENFORCE_LT(
        idx, op_proto->outputs().size(),
        platform::errors::OutOfRange(
            "The index should be less than the size of outputs of "
            "operator %s, but got index is %d and size is %d",
            op_.Type(), idx, op_proto->outputs().size()));
    return op_proto->outputs()[idx].name();
  }

713 714
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
X
Xin Pan 已提交
715 716
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
    PADDLE_ENFORCE_NE(
        in_it, ctx_.inputs.end(),
        platform::errors::NotFound("Input %s does not exist.", in));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output %s does not exist.", out));
    PADDLE_ENFORCE_LT(i, in_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of input dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          in_it->second.size(), i));
    PADDLE_ENFORCE_LT(j, out_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of output dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          out_it->second.size(), j));
X
Xin Pan 已提交
733 734 735

    Variable* in_var = in_it->second[i];
    Variable* out_var = out_it->second[j];
736

737 738 739 740 741
    PADDLE_ENFORCE_EQ(
        in_var->Type(), out_var->Type(),
        platform::errors::InvalidArgument(
            "The type of input (%s) and output (%s) are inconsistent.", in,
            out));
742 743 744 745 746 747 748 749 750 751 752 753

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
754
      PADDLE_THROW(platform::errors::Unimplemented(
755
          "Currently, the input type of ShareDim only can be LoDTensor "
756
          "or SelectedRows."));
757 758 759
    }
  }

H
hong 已提交
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
  void ShareAllLoD(const std::string& in,
                   const std::string& out) const override {
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(),
                      platform::errors::NotFound(
                          "Input [%s] found error in Op [%s]", in, op_.Type()));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output [%s] found error in Op [%s]", out,
                                   op_.Type()));

    auto& in_var_list = in_it->second;
    auto& out_var_list = out_it->second;

    PADDLE_ENFORCE_EQ(
        in_var_list.size(), out_var_list.size(),
        platform::errors::PreconditionNotMet(
T
tianshuo78520a 已提交
778
            "Op [%s]: Input var size should be equal with output var size",
H
hong 已提交
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
            op_.Type()));

    auto& out_var_names = op_.Outputs(out);

    for (size_t i = 0; i < in_var_list.size(); ++i) {
      if (out_var_names[i] == framework::kEmptyVarName) {
        continue;
      }

      Variable* in_var = in_var_list[i];
      if (!in_var->IsType<LoDTensor>()) return;
      Variable* out_var = out_var_list[i];
      PADDLE_ENFORCE_EQ(out_var->IsType<LoDTensor>(), true,
                        platform::errors::PreconditionNotMet(
                            "The %d-th output of Output(%s) must be LoDTensor.",
                            i, out_var_names[i]));
      auto& in_tensor = in_var->Get<LoDTensor>();
      auto* out_tensor = out_var->GetMutable<LoDTensor>();
      out_tensor->set_lod(in_tensor.lod());
#ifdef PADDLE_WITH_MKLDNN
      if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
        out_tensor->set_layout(in_tensor.layout());
    }
  }

Q
Qiao Longfei 已提交
805 806
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
807 808
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
    PADDLE_ENFORCE_NE(
        in_it, ctx_.inputs.end(),
        platform::errors::NotFound("Input %s does not exist.", in));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output %s does not exist.", out));
    PADDLE_ENFORCE_LT(i, in_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of input dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          in_it->second.size(), i));
    PADDLE_ENFORCE_LT(j, out_it->second.size(),
                      platform::errors::InvalidArgument(
                          "The index of output dimension is out of range, "
                          "excepted index less than %zu, but received %zu.",
                          out_it->second.size(), j));
X
Xin Pan 已提交
825 826

    Variable* in_var = in_it->second.at(i);
Q
Qiao Longfei 已提交
827
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
828
    Variable* out_var = out_it->second.at(j);
829 830 831 832
    PADDLE_ENFORCE_EQ(
        out_var->IsType<LoDTensor>(), true,
        platform::errors::InvalidArgument(
            "The %zu-th output of Output(%s) must be LoDTensor.", j, out));
833
    auto& in_tensor = in_var->Get<LoDTensor>();
Q
Qiao Longfei 已提交
834 835
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
836

M
mozga-intel 已提交
837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
856 857
  }

858
  int32_t GetLoDLevel(const std::string& in, size_t i = 0) const override {
859
    PADDLE_THROW(platform::errors::PreconditionNotMet(
860
        "GetLoDLevel is only used in compile time. The calculation of "
861
        "output's actual lod is different among operators so that should be "
862
        "set in the runtime kernel."));
863 864
  }

865 866
  void SetLoDLevel(const std::string& out, int32_t lod_level,
                   size_t j = 0) const override {
867
    PADDLE_THROW(platform::errors::PreconditionNotMet(
868
        "SetLoDLevel is only used in compile time. The calculation of "
869
        "output's actual lod is different among operators so that should be "
870
        "set in the runtime kernel."));
C
chengduo 已提交
871 872
  }

873 874
  bool IsRuntime() const override { return true; }

875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
  // TODO(paddle-dev): Can this be template?
  std::vector<InferShapeVarPtr> GetInputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = InputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

  std::vector<InferShapeVarPtr> GetOutputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = OutputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

X
Xin Pan 已提交
894 895
  DDim GetInputDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
896 897 898 899 900
    PADDLE_ENFORCE_EQ(
        vars.size(), 1UL,
        platform::errors::InvalidArgument(
            "Input(%s) should hold one element, but now it holds %zu elements.",
            name, vars.size()));
X
Xin Pan 已提交
901 902 903 904 905 906 907 908
    return this->GetDim(vars[0]);
  }

  std::vector<DDim> GetInputsDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    return GetDims(vars);
  }

X
Xin Pan 已提交
909 910 911 912 913 914 915 916 917 918
  std::vector<proto::VarType::Type> GetInputsVarType(
      const std::string& name) const override {
    return GetVarTypes(InputVars(name));
  }

  std::vector<proto::VarType::Type> GetOutputsVarType(
      const std::string& name) const override {
    return GetVarTypes(OutputVars(name));
  }

X
Xin Pan 已提交
919 920
  void SetOutputDim(const std::string& name, const DDim& dim) override {
    auto& vars = OutputVars(name);
921 922 923 924 925
    PADDLE_ENFORCE_EQ(
        vars.size(), 1UL,
        platform::errors::InvalidArgument("Output(%s) should hold one element, "
                                          "but now it holds %zu elements.",
                                          name, vars.size()));
X
Xin Pan 已提交
926 927 928 929 930 931 932 933 934
    SetDim(vars[0], dim);
  }

  void SetOutputsDim(const std::string& name,
                     const std::vector<DDim>& dims) override {
    auto& vars = OutputVars(name);
    SetDims(vars, dims);
  }

935
 protected:
X
Xin Pan 已提交
936
  DDim GetDim(Variable* var) const {
937 938
    PADDLE_ENFORCE_NOT_NULL(
        var, platform::errors::InvalidArgument("Input variable is nullptr."));
939 940 941 942 943
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
944 945 946 947
      PADDLE_THROW(platform::errors::InvalidArgument(
          "Only LoDTensor or SelectedRows support 'GetDim', but input "
          "Variable's type is %s.",
          ToTypeName(var->Type())));
F
fengjiayi 已提交
948 949 950
    }
  }

X
Xin Pan 已提交
951 952 953 954 955 956 957 958
  std::vector<DDim> GetDims(const std::vector<Variable*>& vars) const {
    std::vector<DDim> ret;
    ret.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(ret),
                   [this](Variable* var) { return this->GetDim(var); });
    return ret;
  }

F
fengjiayi 已提交
959
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
960 961
    PADDLE_THROW(platform::errors::PreconditionNotMet(
        "GetRepeatedDims method only ban be used in compile time."));
962 963
  }

X
Xin Pan 已提交
964
  void SetDim(Variable* var, const DDim& dim) {
965 966 967 968 969
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
970 971 972 973
      PADDLE_THROW(platform::errors::Unimplemented(
          "Variable type error, expect LoDTensor or SelectedRows, but received "
          "(%s).",
          ToTypeName(var->Type())));
X
Xin Pan 已提交
974 975 976 977 978 979
    }
  }

  void SetDims(const std::vector<Variable*>& vars,
               const std::vector<DDim>& dims) {
    size_t length = vars.size();
980 981 982 983 984 985
    PADDLE_ENFORCE_EQ(length, dims.size(),
                      platform::errors::InvalidArgument(
                          "The number of input variables do not match the "
                          "number of input dimensions, the number of variables "
                          "is %zu, the number of dimensions is %zu.",
                          length, dims.size()));
X
Xin Pan 已提交
986 987 988 989 990
    for (size_t i = 0; i < length; ++i) {
      if (vars[i] == nullptr) {
        continue;
      }
      SetDim(vars[i], dims[i]);
991 992 993
    }
  }

F
fengjiayi 已提交
994 995
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
996 997
    PADDLE_THROW(platform::errors::PreconditionNotMet(
        "SetRepeatedDims method only can be used in compile time."));
F
fengjiayi 已提交
998 999
  }

X
Xin Pan 已提交
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
  std::vector<proto::VarType::Type> GetVarTypes(
      const std::vector<Variable*>& vars) const {
    std::vector<proto::VarType::Type> retv;
    retv.resize(vars.size());
    std::transform(vars.begin(), vars.end(), retv.begin(),
                   std::bind(std::mem_fn(&RuntimeInferShapeContext::GetVarType),
                             this, std::placeholders::_1));
    return retv;
  }

  proto::VarType::Type GetVarType(Variable* var) const {
1011 1012 1013
    return ToVarType(var->Type());
  }

1014 1015 1016
 private:
  const std::vector<Variable*>& InputVars(const std::string& name) const {
    auto it = ctx_.inputs.find(name);
1017 1018 1019 1020
    PADDLE_ENFORCE_NE(
        it, ctx_.inputs.end(),
        platform::errors::NotFound(
            "Operator (%s) does not have the input (%s).", op_.Type(), name));
1021 1022 1023 1024 1025
    return it->second;
  }

  const std::vector<Variable*>& OutputVars(const std::string& name) const {
    auto it = ctx_.outputs.find(name);
1026 1027 1028 1029
    PADDLE_ENFORCE_NE(
        it, ctx_.outputs.end(),
        platform::errors::NotFound(
            "Operator (%s) does not have the outputs (%s).", op_.Type(), name));
1030
    return it->second;
F
fengjiayi 已提交
1031 1032
  }

1033
  const OperatorBase& op_;
X
Xin Pan 已提交
1034
  const RuntimeContext& ctx_;
1035 1036
};

1037 1038
static void CheckTensorNANOrInf(const std::string& op_type,
                                const std::string& name,
C
chengduoZH 已提交
1039 1040 1041 1042
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
Y
Yu Yang 已提交
1043 1044
  if (tensor.type() != proto::VarType::FP32 &&
      tensor.type() != proto::VarType::FP64) {
C
chengduoZH 已提交
1045 1046
    return;
  }
1047 1048 1049 1050 1051 1052 1053 1054
  PADDLE_ENFORCE_NE(
      framework::TensorContainsInf(tensor), true,
      platform::errors::Fatal("Operator %s output Tensor %s contains Inf.",
                              op_type, name));
  PADDLE_ENFORCE_NE(
      framework::TensorContainsNAN(tensor), true,
      platform::errors::Fatal("Operator %s output Tensor %s contains NAN.",
                              op_type, name));
C
chengduoZH 已提交
1055 1056
}

1057 1058
bool OperatorWithKernel::SupportsMKLDNN(
    const proto::VarType::Type data_type) const {
1059 1060
  auto& op_kernels = OperatorWithKernel::AllOpKernels().at(type_);
  return std::any_of(op_kernels.begin(), op_kernels.end(),
1061
                     [data_type](OpKernelMap::const_reference kern_pair) {
1062 1063
                       return platform::is_cpu_place(kern_pair.first.place_) &&
                              kern_pair.first.library_type_ ==
1064 1065
                                  LibraryType::kMKLDNN &&
                              kern_pair.first.data_type_ == data_type;
1066 1067 1068
                     });
}

1069 1070
bool OperatorWithKernel::CanMKLDNNBeUsed(const framework::ExecutionContext& ctx,
                                         proto::VarType::Type data_type) const {
1071 1072
  bool use_mkldnn_ctx =
      ctx.Attr<bool>("use_mkldnn") && platform::is_cpu_place(ctx.GetPlace());
1073
  return use_mkldnn_ctx && this->SupportsMKLDNN(data_type);
1074 1075
}

B
baojun-nervana 已提交
1076
void OperatorWithKernel::RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
1077 1078
                                           const platform::Place& place,
                                           const RuntimeContext& ctx) const {
1079
  RuntimeInferShapeContext infer_shape_ctx(*this, ctx);
B
baojun-nervana 已提交
1080 1081 1082
  this->InferShape(&infer_shape_ctx);
}

L
luotao1 已提交
1083 1084
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
L
luotao1 已提交
1085 1086
  // To reduce the elapsed time of HasAttr, we use bool variable to record the
  // result of HasAttr.
1087 1088 1089
  if (!enable_cache_runtime_context_ && HasAttr(kEnableCacheRuntimeContext))
    enable_cache_runtime_context_ = true;
  if (!all_kernels_must_compute_runtime_shape_ &&
L
luotao1 已提交
1090
      HasAttr(kAllKernelsMustComputeRuntimeShape))
1091
    all_kernels_must_compute_runtime_shape_ = true;
1092
  const Scope* cur_scope = &scope;
1093
  if (!enable_cache_runtime_context_) {
L
luotao1 已提交
1094 1095
    RuntimeContext ctx(Inputs(), Outputs(), scope);
    RunImpl(scope, place, &ctx);
1096
    pre_scope_ = cur_scope;
L
luotao1 已提交
1097
  } else {
1098 1099 1100 1101 1102 1103
    if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
      std::lock_guard<std::mutex> lock(cache_update_mutex_);
      if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
        runtime_ctx_.reset(new RuntimeContext(Inputs(), Outputs(), scope));
        pre_scope_ = cur_scope;
      }
L
luotao1 已提交
1104 1105 1106 1107 1108 1109 1110 1111
    }
    RunImpl(scope, place, runtime_ctx_.get());
  }
}

void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place,
                                 RuntimeContext* runtime_ctx) const {
Y
Yu Yang 已提交
1112
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
1113
  auto* dev_ctx = pool.Get(place);
1114

1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
#ifdef PADDLE_WITH_ASCEND_CL
  // NOTE(wangxi): nan/inf cannot be detected on NPU by checking the variable
  // values, but only through special `float_status` to checks whether
  // the operation is overflow. More about `float_status`, see:
  // https://gitee.com/ascend/modelzoo/issues/I3NF8V?from=project-issue
  if (FLAGS_check_nan_inf) {
    framework::details::NPUAllocAndClearFloatStatus(*this, scope, place);
  }
#endif

1125 1126 1127 1128 1129 1130 1131 1132 1133
  auto exe_ctx = ExecutionContext(*this, scope, *dev_ctx, *runtime_ctx);

  // TODO(chenweihang): Now we are still reusing a lot of the original fluid
  // implementation, this is a gradual replacement process
  // TODO(chenweihang): in the first phase of project, we only support CPU, CUDA
  // and RCOM backend, the XPU, NPU and MKLDNN will be supported in the second
  // phase
  if (FLAGS_run_pten_kernel &&
      pten::KernelFactory::Instance().HasCompatiblePtenKernel(type_)) {
1134
    if (pt_kernel_signature_ == nullptr || pt_kernel_ == nullptr) {
1135 1136 1137 1138 1139 1140 1141 1142
      ChoosePtenKernel(exe_ctx);
    }
    run_pten_kernel_ = pt_kernel_->IsValid();
  }
  if (!run_pten_kernel_) {
    if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
      ChooseKernel(exe_ctx);
    }
1143 1144
  }

Y
yuyang18 已提交
1145 1146
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
1147 1148
  Scope* transfer_scope = nullptr;
  {
1149
    platform::RecordEvent record_event("prepare_data",
1150
                                       platform::EventRole::kInnerOp);
1151 1152 1153 1154
    if (need_prepare_data_) {
      transfer_scope = PrepareData(scope, *kernel_type_,
                                   &transfered_inplace_vars, runtime_ctx);
    }
1155
  }
Y
yuyang18 已提交
1156 1157 1158 1159
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

1160 1161
  if (!(kernel_type_->place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(kernel_type_->place_);
1162
  }
Q
QI JUN 已提交
1163

1164
  if (!all_kernels_must_compute_runtime_shape_) {
1165
    platform::RecordEvent record_event("infer_shape",
1166
                                       platform::EventRole::kInnerOp);
1167
    RuntimeInferShapeContext infer_shape_ctx(*this, *runtime_ctx);
1168 1169
    this->InferShape(&infer_shape_ctx);
  }
1170 1171 1172 1173 1174

  if (FLAGS_enable_unused_var_check) {
    GetThreadLocalUsedVarNameSet()->clear();
  }

X
clean  
Xin Pan 已提交
1175 1176
  // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
  // not Scope. Imperative mode only pass inputs and get outputs.
1177
  {
1178
    platform::RecordEvent record_event("compute",
1179
                                       platform::EventRole::kInnerOp);
1180
    if (run_pten_kernel_) {
1181 1182 1183 1184 1185
      if (pt_kernel_context_ == nullptr) {
        pt_kernel_context_.reset(new pten::KernelContext());
      }
      BuildPtenKernelContext(*runtime_ctx, dev_ctx);
      (*pt_kernel_)(pt_kernel_context_.get());
1186 1187 1188

      WriteBackToOutputs(runtime_ctx);

1189
      pt_kernel_context_->ClearData();
1190 1191 1192 1193
    } else {
      (*kernel_func_)(
          ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx));
    }
1194
  }
D
dzhwinter 已提交
1195

Y
yuyang18 已提交
1196
  if (!transfered_inplace_vars.empty()) {
T
tianshuo78520a 已提交
1197
    // there is inplace variable has been transferred.
Y
yuyang18 已提交
1198
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
1199
  }
1200 1201 1202 1203 1204 1205 1206

  // See [ Why need handle complex gradient to real gradient? ]
  // Only handle the case where the current kernel data type is complex
  if (framework::IsComplexType(kernel_type_->data_type_)) {
    HandleComplexGradToRealGrad(scope, runtime_ctx);
  }

1207 1208 1209 1210 1211 1212 1213 1214
  if (FLAGS_enable_unused_var_check) {
    // skip op that uses mkldnn because it has different memory reuse strategy.
    // use attr here because some GradMakers (like ActivationGradOpMaker) add
    // input when use_mkldnn=true;
    if (!(HasAttr("use_mkldnn") && Attr<bool>("use_mkldnn"))) {
      CheckUnusedVar(*this, scope);
    }
  }
1215

D
dzhwinter 已提交
1216
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
1217
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
1218
    dev_ctx->Wait();
L
Leo Chen 已提交
1219 1220 1221
#if defined(PADDLE_WITH_CUDA)
    PADDLE_ENFORCE_CUDA_SUCCESS(cudaGetLastError());
    VLOG(4) << "Operator(" << Type() << "): context wait and get last error";
1222 1223 1224 1225
#endif
#if defined(PADDLE_WITH_HIP)
    PADDLE_ENFORCE_CUDA_SUCCESS(hipGetLastError());
    VLOG(4) << "Operator(" << Type() << "): context wait and get last error";
L
Leo Chen 已提交
1226
#endif
D
dzhwinter 已提交
1227
  }
C
chengduoZH 已提交
1228 1229

  if (FLAGS_check_nan_inf) {
W
WangXi 已提交
1230
    framework::details::CheckOpHasNanOrInf(*this, exec_scope, place);
C
chengduoZH 已提交
1231
  }
1232 1233 1234 1235 1236 1237 1238

  // To solve issue #15032, have a discussion with @Luotao for cpu inference,
  // do not cache transfer scope, hence in this case delete transfer scope
  // after run to avoid memory leak
  if (transfer_scope && !run_by_executor_ && !enable_cache_transfer_scope_) {
    scope.DeleteScope(transfer_scope);
  }
Q
Qiao Longfei 已提交
1239
}
X
Xin Pan 已提交
1240

1241 1242 1243
OpKernelType OperatorWithKernel::InnerGetExpectedKernelType(
    const ExecutionContext& ctx) const {
  auto& dev_ctx = ctx.device_context();
L
Liu Yiqun 已提交
1244

1245
  auto expected_kernel_key = this->GetExpectedKernelType(ctx);
1246 1247 1248
  if (HasAttr("op_device")) {
    if (Attr<std::string>("op_device") == "cpu") {
      expected_kernel_key.place_ = platform::CPUPlace();
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
    } else if (Attr<std::string>("op_device").find("gpu") !=
               std::string::npos) {
      auto device = Attr<std::string>("op_device");
      size_t pos = device.find(':');
      if (pos != std::string::npos) {
        device = device.substr(0, pos);
        LOG_FIRST_N(WARNING, 1)
            << "Device index is only supported under pipeline parallelism, "
            << "so it will be ignored.";
      }
1259 1260 1261
      // when the Op that only has CPUKernel is assigned to GPU, the CPUKernel
      // will be executed and a warning will be given at the same time.
      if (SupportGPU()) {
1262
        expected_kernel_key.place_ = dev_ctx.GetPlace();
B
Baibaifan 已提交
1263
      } else if (SupportNPU()) {
1264
        expected_kernel_key.place_ = dev_ctx.GetPlace();
1265 1266 1267 1268 1269 1270 1271 1272
      } else {
        expected_kernel_key.place_ = platform::CPUPlace();
        LOG_FIRST_N(WARNING, 1)
            << "Op(" << type_
            << ") has no CUDA implementation. It will be assigned to CPUPlace.";
      }
    }
  }
C
cc 已提交
1273 1274
  VLOG(3) << "op type:" << type_
          << ", expected_kernel_key:" << expected_kernel_key;
1275 1276 1277 1278 1279 1280
  return expected_kernel_key;
}

void OperatorWithKernel::ChoosePtenKernel(const ExecutionContext& ctx) const {
  pt_kernel_signature_.reset(
      new KernelSignature(std::move(this->GetExpectedPtenKernelArgs(ctx))));
C
Chen Weihang 已提交
1281
  VLOG(6) << KernelSignatureToString(*pt_kernel_signature_.get());
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292

  kernel_type_.reset(
      new OpKernelType(std::move(InnerGetExpectedKernelType(ctx))));

  auto pt_kernel_name = pten::KernelName(pt_kernel_signature_->name);
  auto pt_kernel_key = TransOpKernelTypeToPtenKernelKey(*kernel_type_.get());
  pt_kernel_.reset(
      new pten::Kernel(pten::KernelFactory::Instance().SelectKernel(
          pt_kernel_name, pt_kernel_key)));

  if (pt_kernel_->IsValid()) {
C
Chen Weihang 已提交
1293
    VLOG(6) << "Static mode ChoosePtenKernel - kernel name: " << pt_kernel_name
1294 1295 1296
            << " | kernel key: " << pt_kernel_key
            << " | kernel: " << *pt_kernel_;
  } else {
C
Chen Weihang 已提交
1297
    VLOG(6) << "Static mode ChoosePtenKernel - kernel `" << pt_kernel_name
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
            << "` not found.";
  }
}

void OperatorWithKernel::ChooseKernel(const ExecutionContext& ctx) const {
  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  PADDLE_ENFORCE_NE(
      kernels_iter, all_op_kernels.end(),
      platform::errors::Unavailable(
          "There are no kernels which are registered in the %s operator.",
          type_));

  OpKernelMap& kernels = kernels_iter->second;

  auto expected_kernel_key = InnerGetExpectedKernelType(ctx);
L
Liu Yiqun 已提交
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325

  auto kernel_iter = kernels.find(expected_kernel_key);
#ifdef PADDLE_WITH_MKLDNN
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
    VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one";
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
1326 1327
#endif
#ifdef PADDLE_WITH_XPU
Q
QingshuChen 已提交
1328 1329 1330 1331
  if (is_xpu_place(expected_kernel_key.place_) &&
      (kernel_iter == kernels.end() ||
       !paddle::platform::is_xpu_support_op(type_, expected_kernel_key) ||
       paddle::platform::is_in_xpu_black_list(type_))) {
1332 1333 1334 1335 1336 1337
    VLOG(3) << "missing XPU kernel: " << type_
            << ", expected_kernel_key:" << expected_kernel_key
            << ", fallbacking to CPU one!";
    expected_kernel_key.place_ = platform::CPUPlace();
    kernel_iter = kernels.find(expected_kernel_key);
  }
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
#endif
#ifdef PADDLE_WITH_ASCEND_CL
  if (kernel_iter == kernels.end() &&
      is_npu_place(expected_kernel_key.place_)) {
    VLOG(3) << "missing NPU kernel: " << type_
            << ", expected_kernel_key:" << expected_kernel_key
            << ", fallbacking to CPU one!";
    expected_kernel_key.place_ = platform::CPUPlace();
    kernel_iter = kernels.find(expected_kernel_key);
  }
L
Liu Yiqun 已提交
1348
#endif
1349 1350 1351 1352
  PADDLE_ENFORCE_NE(kernel_iter, kernels.end(),
                    platform::errors::NotFound(
                        "Operator (%s) does not have kernel for %s.", type_,
                        KernelTypeToString(expected_kernel_key)));
L
Liu Yiqun 已提交
1353

1354 1355 1356 1357 1358
  std::lock_guard<std::mutex> lock(cache_update_mutex_);
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
    kernel_type_.reset(new OpKernelType(expected_kernel_key));
    kernel_func_.reset(new OpKernelFunc(kernel_iter->second));
  }
L
Liu Yiqun 已提交
1359 1360
}

Y
yuyang18 已提交
1361 1362 1363 1364
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
M
minqiyang 已提交
1365
    VLOG(3) << "share inplace var " + var_name + " back to it's original scope";
C
chengduo 已提交
1366
    auto* origin_var = scope.FindVar(var_name);
1367 1368 1369
    PADDLE_ENFORCE_NOT_NULL(origin_var,
                            platform::errors::InvalidArgument(
                                "The variable[%s] is nullptr.", var_name));
C
chengduo 已提交
1370
    auto* original_tensor =
C
chengduo 已提交
1371
        GetMutableLoDTensorOrSelectedRowsValueFromVar(origin_var);
C
chengduo 已提交
1372
    auto* var = transfer_scope.FindVar(var_name);
1373 1374
    PADDLE_ENFORCE_NOT_NULL(var, platform::errors::InvalidArgument(
                                     "The variable[%s] is nullptr.", var_name));
C
chengduo 已提交
1375
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1376
    auto original_dims = original_tensor->dims();
Y
yuyang18 已提交
1377
    original_tensor->ShareDataWith(*transformed_tensor);
B
Baibaifan 已提交
1378 1379 1380 1381 1382
    // In order to solve the problem that the output latitude of NPU reshape
    // operator is not changed when inplace.
    if (type_ != "reshape2" && type_ != "reshape2_grad") {
      original_tensor->Resize(original_dims);
    }
Y
yuyang18 已提交
1383 1384 1385
  }
}

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
void OperatorWithKernel::HandleComplexGradToRealGrad(
    const Scope& scope, RuntimeContext* ctx) const {
  for (auto& var_name_item : Outputs()) {
    std::vector<Variable*>& output_vars = ctx->outputs[var_name_item.first];
    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      // 1. find grad_var & check whether is complex tensor
      auto var_name = var_name_item.second[i];
      auto orig_var_name = GradOriginalVarName(var_name);
      // only focus on gradient var
      if (var_name == orig_var_name) {
        continue;
      }
      auto* grad_var = output_vars[i];
      // skip nullptr var
      if (grad_var == nullptr) {
        continue;
      }
      // don't process LoDTensorArray temporarily,
      // add support if necessary for complex number calculations in the future
      if (!VarIsTensor(*grad_var)) {
        continue;
      }
      auto* grad_tensor =
          GetMutableLoDTensorOrSelectedRowsValueFromVar(grad_var);
      // skip nullptr tensor
      if (grad_tensor == nullptr || !grad_tensor->IsInitialized()) {
        continue;
      }
      // only focus on complex dtype now
      auto src_type = grad_tensor->type();
      if (!IsComplexType(src_type)) {
        continue;
      }

      // 2. find forward var & check whether need to cast
      auto* var = scope.FindVar(orig_var_name);
      // if forward var not exists, do nothing
      if (var == nullptr) {
        continue;
      }
      if (!VarIsTensor(*var)) {
        continue;
      }
      const auto* tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
      PADDLE_ENFORCE_NOT_NULL(
          tensor,
          platform::errors::Unavailable(
              "Forward tensor is nullptr when handle complex data to real."));
      // only need record type, the allocation may have been released
      auto dst_type = tensor->saved_type();
      // only focus on real dtype and need casting
      if (IsComplexType(dst_type)) {
        continue;
      }

      // 3. cast complex grad to real grad
      VLOG(6) << "Transform " << framework::DataTypeToString(src_type)
              << " var `" << var_name << "` to "
              << framework::DataTypeToString(dst_type)
              << " real var in static graph.";
      Tensor out;
      TransComplexToReal(dst_type, src_type, *grad_tensor, &out);
      SetTensorToVariable(*grad_var, out, grad_var);
    }
  }
}

X
Xin Pan 已提交
1453
Scope* OperatorWithKernel::PrepareData(
Y
yuyang18 已提交
1454
    const Scope& scope, const OpKernelType& expected_kernel_key,
X
Xin Pan 已提交
1455 1456
    std::vector<std::string>* transfered_inplace_vars,
    RuntimeContext* ctx) const {
Y
yuyang18 已提交
1457
  Scope* new_scope = nullptr;
S
sneaxiy 已提交
1458

1459
  const std::unordered_set<std::string>* no_buffer_ins = nullptr;
S
sneaxiy 已提交
1460 1461 1462 1463
  if (info_) {
    auto& no_buffer_inferer = info_->NoNeedBufferVarsInferer();
    // Some op may not register NoNeedBufferVarsInferer
    if (no_buffer_inferer) {
1464 1465
      no_buffer_ins = &(no_buffer_inferer(Inputs(), Outputs(), Attrs()));
      if (no_buffer_ins->empty()) no_buffer_ins = nullptr;
S
sneaxiy 已提交
1466 1467 1468
    }
  }

Y
yuyang18 已提交
1469
  for (auto& var_name_item : Inputs()) {
1470 1471
    bool should_skip_input =
        no_buffer_ins && no_buffer_ins->count(var_name_item.first) > 0;
S
sneaxiy 已提交
1472

X
Xin Pan 已提交
1473 1474 1475 1476
    std::vector<Variable*>& input_vars = ctx->inputs[var_name_item.first];

    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      auto& var_name = var_name_item.second[i];
X
Xin Pan 已提交
1477
      auto* var = input_vars[i];
X
Xin Pan 已提交
1478

Y
yuyang18 已提交
1479
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
1480
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
1481 1482 1483
        continue;
      }

C
chengduo 已提交
1484
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499

      // When no_buffer_ins then checking of Tensor::holder_ is
      // not a thread safe. And for infershape scenario checks
      // to be omitted are not really needed
      if (should_skip_input == true) {
#ifdef PADDLE_WITH_MKLDNN
        // Var without buffer may be needed
        // for some situation like InferShape().
        // In this situation We cannot skip Var analysis, as
        // MKL-DNN shape of Var may differ from kNHWC Var
        // In such situation corressponding resized Var
        // has to be created and registered
        if ((tensor_in->layout() == DataLayout::kMKLDNN) &&
            (var->IsType<LoDTensor>() == true) &&
            (expected_kernel_key.data_layout_ != DataLayout::kMKLDNN) &&
1500 1501
            (paddle::platform::MKLDNNDeviceContext::tls()
                 .get_cur_paddle_data_layout() == DataLayout::kNHWC)) {
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
          // Mixed execution : MKL-DNN and GPU is not supported!
          if (!new_scope) {
            new_scope = &scope.NewScope();
          }
          auto* trans_var = new_scope->Var(var_name);
          input_vars[i] = trans_var;
          auto out = trans_var->GetMutable<LoDTensor>();
          out->Resize(tensor_in->dims());
          platform::MatchShapeToLayout(out, tensor_in->layout(),
                                       DataLayout::kNHWC);
          VLOG(7) << "Created reshaped dummy input based on MKL-DNN Tensor , "
                     "but kNHWC layout"
                  << var_name_item.first << " in Operator " << type_;
        } else {
          VLOG(7) << "Skip scanning input " << var_name_item.first
                  << " in Operator " << type_;
        }
#endif
        continue;
      }

Y
yuyang18 已提交
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

M
minqiyang 已提交
1534 1535
      VLOG(3) << "Transform Variable " << var_name << " from "
              << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
1536

1537 1538 1539
      // In the inference scenerio, the scopes will be reused across the
      // batches, so the `new_scope` here will result in GPU memroy explosion
      // over the  running of operators.
1540
      // We use a thread_local cache to fix that issue, the key in the cache is
1541 1542 1543 1544 1545
      // the combination of the `scope` argument, from_kernel_type,
      // target_kernel_type.
      // Have a discussion with @Superjomn or the inference developers if some
      // changes on this logic for this macro might not tested on the other
      // scenerios.
1546 1547
      // If this op is not called by an Executor or ParallelExecutor, it should
      // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and
1548
      // variables, that behavior a lot different.
1549 1550 1551 1552 1553 1554 1555 1556 1557
      //
      // To solve issue #15032, have a discussion with @Luotao for cpu
      // inference, for all cpu kernels cases without GPU participation, here
      // not do transfer scope caching, and cpu inference performance is not
      // impacted by test.
      enable_cache_transfer_scope_ = false;
      if (!run_by_executor_ &&
          (platform::is_gpu_place(kernel_type_for_var.place_) ||
           platform::is_gpu_place(expected_kernel_key.place_))) {
1558 1559
        new_scope = TryCreateTransferScope(kernel_type_for_var,
                                           expected_kernel_key, &scope);
1560
        enable_cache_transfer_scope_ = true;
1561
      }
1562
      if (!new_scope) {
Y
yuyang18 已提交
1563 1564
        new_scope = &scope.NewScope();
      }
1565 1566 1567 1568
      // For inference, if a gpu model has an op which could only run on CPU,
      // each result of different input will be the same with the first one.
      // The reason is that if a gpu tensor is the input of a cpu kernel,
      // we will create a new cpu tensor in new scope.
1569
      // However, if enable_cache_runtime_context_, we get the cpu tensor each
1570 1571
      // time, not the gpu tensor. Thus, we set pre_scope_ = nullptr
      // to trigger `new RuntimeContext()` in RunImpl().
1572
      if (enable_cache_runtime_context_) {
1573 1574
        pre_scope_ = nullptr;
      }
L
Leo Chen 已提交
1575 1576

      // Create new var with the same name in transfer scopes
Y
yuyang18 已提交
1577
      auto* trans_var = new_scope->Var(var_name);
X
fix  
Xin Pan 已提交
1578
      input_vars[i] = trans_var;
L
Leo Chen 已提交
1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595

      // Find if inplace exists between input and output
      // If inplace exists, set the new created var to inplaced output, and
      // record its name in transfered_inplace_vars.
      for (auto& pair : Outputs()) {
        for (size_t j = 0; j < pair.second.size(); ++j) {
          if (pair.second[j] == var_name) {
            VLOG(4) << "Found inplace between input(" << var_name_item.first
                    << ") and output(" << pair.first
                    << "), the variable name is " << var_name;
            ctx->outputs[pair.first][j] = trans_var;
            transfered_inplace_vars->emplace_back(var_name);
          }
        }
      }

      // Do transfer
Y
yuyang18 已提交
1596
      Tensor out;
Y
yuyang18 已提交
1597
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
1598 1599 1600
      SetTensorToVariable(*var, out, trans_var);
    }
  }
L
Leo Chen 已提交
1601

1602 1603 1604 1605 1606 1607
  // If pre_scope = &scope, it means that scope is cached and the op is not in
  // while block. If new_scope = nullptr, it means that for each input of this
  // Op, there is no need to do PrepareData. So PrepareData could be skipped at
  // the rest iterations to save the elapsed time.
  // We do not support skipping PrepareData in while block, because the Op's
  // input may be changed by subsequent Ops, which may cause an error.
W
wenbin 已提交
1608 1609 1610 1611 1612 1613

  // For inference, ops that behind conditional branch aren't supported well,
  // so disable prepare optimization conservatively.
  bool force_prepare_data = HasAttr("inference_force_prepare_data") &&
                            Attr<bool>("inference_force_prepare_data");
  if (pre_scope_ == &scope && new_scope == nullptr && !force_prepare_data) {
1614 1615
    need_prepare_data_ = false;
  }
Y
yuyang18 已提交
1616 1617 1618

  return new_scope;
}
Q
Qiao Longfei 已提交
1619

1620
void OperatorWithKernel::ParseInputDataType(
1621
    const std::vector<Variable*>& vars, const std::string& name,
1622
    proto::VarType::Type* data_type) const {
1623
  proto::VarType::Type default_data_type =
1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
      static_cast<proto::VarType::Type>(-1);
  for (size_t i = 0; i < vars.size(); ++i) {
    const Variable* var = vars[i];
    if (var != nullptr) {
      const Tensor* t = nullptr;
      if (var->IsType<Tensor>()) {
        t = &var->Get<Tensor>();
      } else if (var->IsType<LoDTensor>()) {
        t = &var->Get<LoDTensor>();
      } else if (var->IsType<SelectedRows>()) {
        t = &(var->Get<SelectedRows>().value());
1635
      } else if (var->IsType<LoDTensorArray>()) {
1636 1637 1638 1639
        auto t_arr = &var->Get<LoDTensorArray>();
        for (size_t j = 0; j < t_arr->size(); j++) {
          if (t_arr->at(j).IsInitialized()) {
            t = &(t_arr->at(j));
1640 1641
          }
        }
1642 1643
      }
      if (t != nullptr) {
1644 1645
        PADDLE_ENFORCE_EQ(
            t->IsInitialized(), true,
1646 1647 1648
            platform::errors::InvalidArgument("The %s Op's Input Variable `%s` "
                                              "contains uninitialized Tensor.",
                                              Type(), name));
1649
        proto::VarType::Type tmp = t->type();
1650 1651 1652 1653 1654 1655 1656 1657 1658
        PADDLE_ENFORCE(tmp == *data_type || *data_type == default_data_type,
                       platform::errors::InvalidArgument(
                           "The DataType of %s Op's duplicable or different "
                           "slot Variable %s must be "
                           "consistent or reigster GetExpectedKernelType. The "
                           "current variable type is (%s), but the "
                           "previous variable type is (%s).",
                           Type(), name, DataTypeToString(tmp),
                           DataTypeToString(*data_type)));
1659 1660 1661 1662 1663 1664
        *data_type = tmp;
      }
    }
  }
}

1665
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
1666
    const ExecutionContext& ctx) const {
1667 1668 1669
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
H
hong 已提交
1670
  for (auto& input : ctx.InNameList()) {
1671 1672
    const std::vector<Variable*> vars = ctx.MultiInputVar(input);
    ParseInputDataType(vars, input, &data_type);
Y
Yu Yang 已提交
1673
  }
1674 1675 1676 1677
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
      platform::errors::NotFound(
          "DataType should be indicated by input Variable at %s.", Type()));
1678 1679 1680 1681 1682 1683 1684 1685
  return data_type;
}

proto::VarType::Type OperatorWithKernel::IndicateVarDataType(
    const ExecutionContext& ctx, const std::string& name) const {
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
1686
  ParseInputDataType(ctx.MultiInputVar(name), name, &data_type);
1687 1688
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
1689 1690 1691 1692 1693
      platform::errors::InvalidArgument(
          "The Input Variable(%s) of (%s) Operator used to determine kernel "
          "data type is empty or not LoDTensor or SelectedRows or "
          "LoDTensorArray.",
          name, Type()));
1694
  return data_type;
Y
Yu Yang 已提交
1695
}
1696

1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
Tensor* OperatorWithKernel::GetTensorFormInputSafely(
    const ExecutionContext& ctx, const std::string& name) const {
  // 1. get variable and check
  // NOTE: only supports signal input var now
  // NOTE: using const_cast is because we don't have method
  // can get single mutable var, and here will not change
  // the var's data, only use some attribute
  Variable* var = const_cast<Variable*>(ctx.InputVar(name));
  PADDLE_ENFORCE_NOT_NULL(
      var,
      platform::errors::NotFound(
          "The variable %s is not found when promote complex types.", name));
  // 2. get tensor and check
  Tensor* t = nullptr;
  if (var->IsType<Tensor>()) {
    t = var->GetMutable<Tensor>();
  } else if (var->IsType<LoDTensor>()) {
    t = var->GetMutable<LoDTensor>();
  } else if (var->IsType<SelectedRows>()) {
    t = var->GetMutable<SelectedRows>()->mutable_value();
  } else {
    PADDLE_THROW(platform::errors::Unimplemented(
        "Unsupported input variable type in complex type promotion."));
  }
  PADDLE_ENFORCE_NOT_NULL(
      t,
      platform::errors::InvalidArgument(
          "The Tensor of variable %s is nullptr when promote complex types."));
  PADDLE_ENFORCE_EQ(t->IsInitialized(), true,
                    platform::errors::InvalidArgument(
                        "The Tensor in the %s Op's Input Variable %s(%s) is "
                        "not initialized.",
                        Type(), name, ctx.InputName(name)));
  return t;
}

/** NOTE(chenweihang): For safety reasons, we now only
 * perform type promotes for binary operations with
 * complex type inputs, which is used to support the
 * paddle quantum function.
 * In other cases, the first input data type is used as
 * the kernel data type.
 */
proto::VarType::Type OperatorWithKernel::IndicateOrPromoteVarDataTypes(
    const ExecutionContext& ctx, const std::string& name1,
    const std::string& name2) const {
  // 1. Get tensor
  auto* tensor_a = GetTensorFormInputSafely(ctx, name1);
  auto* tensor_b = GetTensorFormInputSafely(ctx, name2);

  // 2. Get two input types
  auto type_a = tensor_a->type();
  auto type_b = tensor_b->type();

  // 3. Get first input type or promote complex types
  auto target_type = PromoteTypesIfComplexExists(type_a, type_b);

  return target_type;
}

1757 1758 1759 1760 1761 1762 1763 1764
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
1765 1766
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
1767 1768
}

1769 1770 1771 1772 1773
KernelSignature OperatorWithKernel::GetExpectedPtenKernelArgs(
    const ExecutionContext& ctx) const {
  return KernelSignatureMap::Instance().Get(Type());
}

1774 1775
void OperatorWithKernel::BuildPtenKernelContext(
    const RuntimeContext& ctx, platform::DeviceContext* dev_ctx) const {
1776 1777 1778 1779 1780 1781 1782
  // TODO(chenweihang): now only work for very simple case,
  // many cases need to be deal with later:
  // 1. the input and output are not tensor
  // 2. the dispensbale, duplicable input and output
  // 3. needless attributes remove
  // 4. use pt Tensor directly
  // 5. kernel input is not DenseTensor
1783
  pt_kernel_context_->SetDeviceContext(dev_ctx);
1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811

  auto& input_names = std::get<0>(pt_kernel_signature_->args);
  auto& attr_names = std::get<1>(pt_kernel_signature_->args);
  auto& output_names = std::get<2>(pt_kernel_signature_->args);

  auto input_defs = pt_kernel_->args_def().input_defs();
  auto attr_defs = pt_kernel_->args_def().attribute_defs();
  auto output_defs = pt_kernel_->args_def().output_defs();

  PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(),
                    platform::errors::InvalidArgument(
                        "The size of inputs_args names (%d) must be equal to "
                        "the size of kernel input_defs (%d).",
                        input_names.size(), input_defs.size()));

  PADDLE_ENFORCE_EQ(output_names.size(), output_defs.size(),
                    platform::errors::InvalidArgument(
                        "The size of outputs_args names (%d) must be equal to "
                        "the size of kernel output_defs (%d).",
                        output_names.size(), output_defs.size()));

  PADDLE_ENFORCE_EQ(attr_names.size(), attr_defs.size(),
                    platform::errors::InvalidArgument(
                        "The size of attribute_args names (%d) must be equal "
                        "to the size of kernel attribute_defs (%d).",
                        attr_names.size(), attr_defs.size()));

  for (size_t i = 0; i < input_names.size(); ++i) {
1812 1813
    auto& in_def = input_defs.at(i);
    auto& ins_vector = ctx.inputs.at(input_names[i]);
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825

    // calcute the start and end index of the input tensors
    size_t start_idx =
        (i == 0 ? 0 : pt_kernel_context_->InputRangeAt(i - 1).second);
    size_t end_idx = start_idx + ins_vector.size();

    // The current size of input/output in pt_kernel_context_ is at least equal
    // the start_idx. For the reason of reusing the allocted of inputs or
    // outputs in pt_kernel_context_, the current size of input/output can be
    // greater then the index of which the tensort wanted to set to, so it will
    // use ReMakePtenDenseTensorFromVar to make pten tensor.
    if (pt_kernel_context_->InputsSize() == start_idx) {
1826 1827 1828 1829 1830 1831
      paddle::SmallVector<std::shared_ptr<pten::TensorBase>> tmp_inputs;
      for (auto* var : ins_vector) {
        tmp_inputs.emplace_back(
            experimental::MakePtenTensorBaseFromVar(*var, in_def));
      }
      pt_kernel_context_->EmplaceBackInputs(std::move(tmp_inputs));
1832
    } else if (pt_kernel_context_->InputsSize() > start_idx) {
1833 1834
      size_t input_size = pt_kernel_context_->InputsSize();
      for (size_t j = 0; j < ins_vector.size(); ++j) {
1835
        if (input_size > start_idx + j) {
1836 1837
          experimental::ReMakePtenDenseTensorFromVar(
              *ins_vector[j], in_def,
1838 1839 1840 1841 1842 1843 1844 1845 1846
              pt_kernel_context_->MutableInputAt<pten::DenseTensor>(start_idx +
                                                                    j));
          // TODO(chentianyu03): When multi input kernel, open this code
          /*
          } else {
            pt_kernel_context_->EmplaceBackInputWithoutSetRange(
                experimental::MakePtenTensorBaseFromVar(*ins_vector[j],
          in_def));
          */
1847 1848 1849
        }
      }
      pt_kernel_context_->MutableInputRangeAt(i) =
1850 1851 1852 1853 1854 1855 1856
          std::make_pair(start_idx, end_idx);
    } else {
      PADDLE_THROW(platform::errors::PreconditionNotMet(
          "Error start index when trying to set new tensor to inputs, start "
          "index is `%d`, but current pt_kernel_context_.inputs.size() is "
          "`%d`.",
          start_idx, pt_kernel_context_->InputsSize()));
1857 1858 1859 1860
    }
  }

  for (size_t i = 0; i < output_names.size(); ++i) {
1861 1862
    auto& out_def = output_defs.at(i);
    auto& outs_vector = ctx.outputs.at(output_names[i]);
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873

    size_t start_idx =
        (i == 0 ? 0 : pt_kernel_context_->OutputRangeAt(i - 1).second);
    size_t end_idx = start_idx + outs_vector.size();

    // The current size of input/output in pt_kernel_context_ is at least equal
    // the start_idx. For the reason of reusing the allocted of inputs or
    // outputs in pt_kernel_context_, the current size of input/output can be
    // greater then the index of which the tensort wanted to set to, so it will
    // use ReMakePtenDenseTensorFromVar to make pten tensor.
    if (pt_kernel_context_->OutputsSize() == start_idx) {
1874 1875 1876 1877 1878 1879
      paddle::SmallVector<std::shared_ptr<pten::TensorBase>> tmp_outputs;
      for (auto* var : outs_vector) {
        tmp_outputs.emplace_back(
            experimental::MakePtenTensorBaseFromVar(var, out_def));
      }
      pt_kernel_context_->EmplaceBackOutputs(std::move(tmp_outputs));
1880
    } else if (pt_kernel_context_->OutputsSize() > start_idx) {
1881 1882
      size_t output_size = pt_kernel_context_->OutputsSize();
      for (size_t j = 0; j < outs_vector.size(); ++j) {
1883
        if (output_size > start_idx + j) {
1884 1885
          experimental::ReMakePtenDenseTensorFromVar(
              outs_vector[j], out_def,
1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
              pt_kernel_context_->MutableOutputAt<pten::DenseTensor>(start_idx +
                                                                     j));

          // TODO(chentianyu03): When multi output kernel, open this code
          /*
          } else {
            pt_kernel_context_->EmplaceBackOutputWithoutSetRange(
                experimental::MakePtenTensorBaseFromVar(outs_vector[j],
          out_def));
              */
1896 1897 1898
        }
      }
      pt_kernel_context_->MutableOutputRangeAt(i) =
1899 1900 1901 1902 1903 1904 1905
          std::make_pair(start_idx, end_idx);
    } else {
      PADDLE_THROW(platform::errors::PreconditionNotMet(
          "Error start index when trying to set new tensor to inputs, start "
          "index is `%d`, but current pt_kernel_context_.outputs.size() is "
          "`%d`.",
          start_idx, pt_kernel_context_->OutputsSize()));
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
    }
  }

  for (size_t i = 0; i < attr_names.size(); ++i) {
    auto& attr = Attrs().at(attr_names[i]);
    if (attr_defs[i].type_index == std::type_index(typeid(pten::Scalar))) {
      // TODO(chenweihang): support other attrs later
      // TODO(zhangyunfei): Scalar should hold scaler type, and we should check
      // attribtue type by attr_defs
      if (std::type_index(attr.type()) == std::type_index(typeid(float))) {
1916
        pt_kernel_context_->EmplaceBackAttr(
1917
            std::move(pten::Scalar(BOOST_GET_CONST(float, attr))));
1918 1919
      } else if (std::type_index(attr.type()) ==
                 std::type_index(typeid(std::string))) {
1920
        pt_kernel_context_->EmplaceBackAttr(
1921
            std::move(pten::Scalar(BOOST_GET_CONST(std::string, attr))));
1922 1923 1924 1925 1926 1927 1928 1929 1930
      } else {
        PADDLE_THROW(platform::errors::Unimplemented(
            "unsupported cast op attribute `%s` to Scalar when construct "
            "KernelContext.",
            attr_names[i]));
      }
    } else {
      // TODO(chenweihang): support other attrs later
      if (attr_defs[i].type_index == std::type_index(typeid(int))) {
1931
        pt_kernel_context_->EmplaceBackAttr(BOOST_GET_CONST(int, attr));
1932
      } else if (attr_defs[i].type_index == std::type_index(typeid(float))) {
1933
        pt_kernel_context_->EmplaceBackAttr(BOOST_GET_CONST(float, attr));
1934
      } else if (attr_defs[i].type_index == std::type_index(typeid(bool))) {
1935
        pt_kernel_context_->EmplaceBackAttr(BOOST_GET_CONST(bool, attr));
1936
      } else if (attr_defs[i].type_index ==
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953
                 std::type_index(typeid(pten::DataType))) {
        auto data_type = pten::TransToPtenDataType(
            static_cast<framework::proto::VarType::Type>(
                BOOST_GET_CONST(int, attr)));
        pt_kernel_context_->EmplaceBackAttr(data_type);
      } else if (attr_defs[i].type_index ==
                 std::type_index(typeid(std::vector<int64_t>))) {
        if (std::type_index(attr.type()) ==
            std::type_index(typeid(std::vector<int>))) {
          // Emplace Back Attr according to the type of Pten_Kernel args.
          const auto& vector_int_attr = BOOST_GET_CONST(std::vector<int>, attr);
          const std::vector<int64_t> vector_int64_attr(vector_int_attr.begin(),
                                                       vector_int_attr.end());
          pt_kernel_context_->EmplaceBackAttr(vector_int64_attr);
        }
        // TODO(YuanRisheng) Need support vector<int64_t> attr

1954 1955 1956 1957 1958 1959 1960 1961 1962 1963
      } else {
        PADDLE_THROW(platform::errors::Unimplemented(
            "unsupported cast op attribute `%s` when construct "
            "KernelContext.",
            attr_names[i]));
      }
    }
  }
}

1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984
void OperatorWithKernel::WriteBackToOutputs(RuntimeContext* ctx) const {
  // auto& input_names = std::get<0>(pt_kernel_signature_->args);
  // auto& attr_names = std::get<1>(pt_kernel_signature_->args);
  auto& output_names = std::get<2>(pt_kernel_signature_->args);

  // pt_kernel_context_

  for (size_t i = 0; i < output_names.size(); ++i) {
    auto& outs_vector = ctx->outputs.at(output_names[i]);

    auto& range_pair = pt_kernel_context_->OutputRangeAt(i);
    auto pten_outs =
        pt_kernel_context_->MutableOutputBetween<pten::DenseTensor>(
            range_pair.first, range_pair.second);

    for (size_t j = 0; j < pten_outs.size(); ++j) {
      experimental::MakeVariableFromPtenTensor(pten_outs[j], outs_vector[j]);
    }
  }
}

Q
Qiao Longfei 已提交
1985
}  // namespace framework
L
liaogang 已提交
1986
}  // namespace paddle