operator.cc 47.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include <gflags/gflags.h>
#include <glog/logging.h>
17

18
#include <algorithm>
P
peizhilin 已提交
19 20
#include <sstream>
#include <string>
S
sneaxiy 已提交
21
#include <unordered_set>
P
peizhilin 已提交
22
#include <vector>
Y
Yi Wang 已提交
23
#include "paddle/fluid/framework/data_transform.h"
W
WangXi 已提交
24
#include "paddle/fluid/framework/details/nan_inf_utils.h"
Y
Yi Wang 已提交
25
#include "paddle/fluid/framework/executor.h"
26
#include "paddle/fluid/framework/lod_tensor.h"
27
#include "paddle/fluid/framework/op_call_stack.h"
28
#include "paddle/fluid/framework/op_proto_maker.h"
29
#include "paddle/fluid/framework/operator.h"
Y
Yi Wang 已提交
30
#include "paddle/fluid/framework/shape_inference.h"
31
#include "paddle/fluid/framework/transfer_scope_cache.h"
32
#include "paddle/fluid/framework/unused_var_check.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/var_type.h"
34
#include "paddle/fluid/platform/profiler.h"
Q
Qiao Longfei 已提交
35

36 37 38 39
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/platform/mkldnn_helper.h"
#endif

D
dzhwinter 已提交
40
DECLARE_bool(benchmark);
41
DECLARE_bool(check_nan_inf);
42
DECLARE_bool(enable_unused_var_check);
Q
Qiao Longfei 已提交
43
DEFINE_int32(inner_op_parallelism, 0, "number of threads for inner op");
P
pkpk 已提交
44 45 46
DEFINE_bool(fast_check_nan_inf, false,
            "Fast checking NAN/INF after each operation. It will be a little"
            "bit slow, much faster than check_nan_inf");
D
dzhwinter 已提交
47

Q
Qiao Longfei 已提交
48 49 50
namespace paddle {
namespace framework {

51 52 53 54 55 56
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
57

58 59
static DDim GetDimsDebug(const Scope& scope, const std::string& name,
                         bool get_actual_dim = false) {
60
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
61 62
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
63 64
  }

M
minqiyang 已提交
65 66 67 68 69 70 71 72 73
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
74 75 76 77 78
  } else {
    return DDim({-1});
  }
}

Q
Qiao Longfei 已提交
79 80 81 82 83 84
static bool VarInited(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

D
dzhwinter 已提交
85 86 87 88 89
static std::string GetDtype(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
90

M
minqiyang 已提交
91 92 93
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
94 95
      return "";
    }
Y
Yu Yang 已提交
96
    return DataTypeToString(tensor.type());
M
minqiyang 已提交
97
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
98 99 100 101
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
Y
Yu Yang 已提交
102
      return DataTypeToString(tensor.type());
Q
Qiao Longfei 已提交
103
    }
D
dzhwinter 已提交
104 105 106 107 108
  } else {
    return "";
  }
}

109 110 111 112 113 114
static int GetRowSize(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
115 116
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
117 118 119 120 121
  }

  return -1;
}

122
static LoD GetLoDDebug(const Scope& scope, const std::string& name) {
Q
Qiao Longfei 已提交
123 124 125 126 127 128 129
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
130 131 132
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    return tensor.lod();
Q
Qiao Longfei 已提交
133 134 135 136 137
  } else {
    return default_lod;
  }
}

X
Xin Pan 已提交
138 139 140 141 142
RuntimeContext::RuntimeContext(const VariableNameMap& innames,
                               const VariableNameMap& outnames,
                               const Scope& scope) {
  for (auto& var_name_item : innames) {
    std::vector<Variable*>& input_vars = inputs[var_name_item.first];
X
Xin Pan 已提交
143
    input_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
144 145 146 147 148 149
    for (auto& var_name : var_name_item.second) {
      input_vars.push_back(scope.FindVar(var_name));
    }
  }
  for (auto& var_name_item : outnames) {
    std::vector<Variable*>& output_vars = outputs[var_name_item.first];
X
Xin Pan 已提交
150
    output_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
151 152 153 154 155 156
    for (auto& var_name : var_name_item.second) {
      output_vars.push_back(scope.FindVar(var_name));
    }
  }
}

157
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
P
peizhilin 已提交
158 159 160
  try {
    VLOG(4) << place << " " << DebugStringEx(&scope);
    if (platform::is_gpu_place(place)) {
161
#ifndef PADDLE_WITH_CUDA
P
peizhilin 已提交
162
      PADDLE_THROW("Cannot run operator on place %s", place);
163
#else
P
peizhilin 已提交
164 165
      auto dev_id = boost::get<platform::CUDAPlace>(place).device;
      platform::SetDeviceId(dev_id);
166
#endif
P
peizhilin 已提交
167
    }
P
peizhilin 已提交
168

169
    {
170 171 172 173 174 175
      // TODO(wangchaochaohu) : refine code to use only one RecordEvent)
      // in order to record different op type cost time
      // and different op name cost time,we set two event.
      platform::RecordEvent op_type_record_event(Type());
      auto op_name = platform::OpName(outputs_, Type());
      platform::RecordEvent op_name_record_event(
176
          op_name, platform::EventRole::kUniqueOp);
P
peizhilin 已提交
177 178
      RunImpl(scope, place);
    }
179

Z
Zhang Ting 已提交
180
    VLOG(3) << GetExecutionPlace(place) << " " << DebugStringEx(&scope);
181
  } catch (platform::EnforceNotMet& exception) {
182
    framework::InsertCallStackInfo(Type(), Attrs(), &exception);
183
    throw std::move(exception);
184 185 186 187 188 189
  } catch (platform::EOFException&) {
    std::rethrow_exception(std::current_exception());
  } catch (std::exception& ex) {
    LOG(WARNING) << Type() << " raises an exception "
                 << platform::demangle(typeid(ex).name()) << ", " << ex.what();
    std::rethrow_exception(std::current_exception());
P
peizhilin 已提交
190
  } catch (...) {
191
    LOG(WARNING) << Type() << " raises an unknown exception";
P
peizhilin 已提交
192
    std::rethrow_exception(std::current_exception());
193
  }
194 195
}

196
bool OperatorBase::HasInputs(const std::string& name) const {
M
minqiyang 已提交
197
  return inputs_.find(name) != inputs_.end();
198 199
}

200
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
201
  auto& ins = Inputs(name);
202 203 204 205 206
  PADDLE_ENFORCE_LE(
      ins.size(), 1UL,
      platform::errors::AlreadyExists(
          "Operator %s's input %s should contain only one variable.", type_,
          name));
Y
Yu Yang 已提交
207
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
208 209
}

Y
Yu Yang 已提交
210 211
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
212
  auto it = inputs_.find(name);
213 214
  PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.",
                 type_, name);
Y
Yu Yang 已提交
215
  return it->second;
Y
Yan Chunwei 已提交
216 217
}

218
bool OperatorBase::HasOutputs(const std::string& name) const {
219
  if (outputs_.find(name) != outputs_.end()) {
220 221 222 223 224 225
    return true;
  } else {
    return false;
  }
}

226
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
227
  auto& outs = Outputs(name);
Y
Yu Yang 已提交
228
  PADDLE_ENFORCE_LE(outs.size(), 1UL,
229 230
                    "Operator %s's output %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
231
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
232 233
}

Y
Yu Yang 已提交
234 235
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
236
  auto it = outputs_.find(name);
237 238
  PADDLE_ENFORCE(it != outputs_.end(),
                 "Operator %s does not have an output called %s.", type_, name);
Y
Yu Yang 已提交
239
  return it->second;
Y
Yan Chunwei 已提交
240 241
}

242
std::string OperatorBase::DebugStringEx(const Scope* scope) const {
Q
Qiao Longfei 已提交
243
  std::stringstream ss;
Y
Yu Yang 已提交
244
  ss << "Op(" << type_ << "), inputs:{";
245

246
  const std::unordered_set<std::string>* no_need_buffer_vars = nullptr;
247 248
  if (info_ && info_->NoNeedBufferVarsInferer()) {
    no_need_buffer_vars =
249 250
        &(Info().NoNeedBufferVarsInferer()(Inputs(), Outputs(), Attrs()));
    if (no_need_buffer_vars->empty()) no_need_buffer_vars = nullptr;
251 252
  }

Y
Yu Yang 已提交
253 254
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
255 256
    bool is_no_need_buffer_var =
        (no_need_buffer_vars && no_need_buffer_vars->count(input.first) > 0);
Y
Yu Yang 已提交
257 258
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
259 260
      auto var_name = input.second[i];
      ss << var_name;
261
      if (scope) {
Q
Qiao Longfei 已提交
262 263 264 265 266 267 268
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
269 270 271
          std::string dtype = is_no_need_buffer_var
                                  ? "unknown_dtype"
                                  : GetDtype(*scope, var_name);
Q
Qiao Longfei 已提交
272
          ss << ":" << dtype;
273 274
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
275
        }
276
      }
Y
Yu Yang 已提交
277 278 279
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
280
    }
Y
Yu Yang 已提交
281
    ss << "]";
Y
Yu Yang 已提交
282 283
    ++it;
    if (it != inputs_.end()) {
284 285
      ss << ", ";
    }
Q
Qiao Longfei 已提交
286
  }
Y
Yu Yang 已提交
287
  ss << "}, outputs:{";
Y
Yu Yang 已提交
288 289
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
290 291
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
292 293
      auto var_name = output.second[i];
      ss << var_name;
294
      if (scope) {
Q
Qiao Longfei 已提交
295 296 297 298 299 300 301
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
302 303
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
304 305
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
306
        }
307
      }
Y
Yu Yang 已提交
308 309 310
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
311
    }
Y
Yu Yang 已提交
312
    ss << "]";
Y
Yu Yang 已提交
313 314
    ++it;
    if (it != outputs_.end()) {
315 316
      ss << ", ";
    }
Q
Qiao Longfei 已提交
317
  }
Y
Yu Yang 已提交
318
  ss << "}.";
Q
Qiao Longfei 已提交
319 320 321
  return ss.str();
}

Y
Yu Yang 已提交
322
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
323 324
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
325
                           const AttributeMap& attrs)
S
sneaxiy 已提交
326 327 328 329 330 331
    : type_(type),
      inputs_(inputs),
      outputs_(outputs),
      attrs_(attrs),
      // NOTE(zjl): why op_info may be nullptr?
      info_(OpInfoMap::Instance().GetNullable(type)) {
H
hong 已提交
332 333 334 335 336 337 338 339
  // In dygraph mode, all the OperatorBase will be constructed by function:
  // framework::OpRegistry::CreateOp(type, {}, {}, {}, false).
  // Inputs, outputs and attrs will be set to empty map
  // to improve the execution efficiency of dygraph.
  if (inputs_.size() > 0 || outputs_.size() > 0) {
    GenerateTemporaryNames();
    CheckAllInputOutputSet();
  }
Y
Yu Yang 已提交
340
}
341

Q
qijun 已提交
342 343
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
344
  for (auto& o : inputs_) {
Q
qijun 已提交
345 346 347 348 349 350
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
351 352 353 354 355 356 357 358 359 360
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
S
sneaxiy 已提交
361
  auto& info = Info();
Y
Yu Yang 已提交
362 363

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
364
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
365 366 367 368 369 370 371 372 373
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
374 375
}

376
void OperatorBase::CheckAllInputOutputSet() const {
S
sneaxiy 已提交
377
  if (info_ == nullptr || info_->proto_ == nullptr) return;
378

S
sneaxiy 已提交
379
  for (auto& in : info_->Proto().inputs()) {
380 381 382 383
    if (!in.dispensable()) {
      PADDLE_ENFORCE(inputs_.find(in.name()) != inputs_.end(),
                     "Operator %s's input, %s, is not set", Type(), in.name());
    }
384 385
  }

S
sneaxiy 已提交
386
  for (auto& out : info_->Proto().outputs()) {
387 388 389 390 391
    if (!out.dispensable()) {
      PADDLE_ENFORCE(outputs_.find(out.name()) != outputs_.end(),
                     "Operator %s's output, %s, is not set", Type(),
                     out.name());
    }
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

B
baojun-nervana 已提交
408
static bool VarIsTensor(const Variable& var) {
C
chengduo 已提交
409
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
410 411
}

C
chengduo 已提交
412
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
413 414 415 416
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
417
  } else {
Y
Yang Yang 已提交
418
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
419
                 ToTypeName(var.Type()));
Q
QI JUN 已提交
420 421 422
  }
}

C
chengduo 已提交
423
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
424
  if (var->IsType<LoDTensor>()) {
425
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
426
  } else if (var->IsType<SelectedRows>()) {
427
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
428
  } else {
Y
Yang Yang 已提交
429
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
430
                 ToTypeName(var->Type()));
Q
QI JUN 已提交
431 432 433
  }
}

434
bool ExecutionContext::HasInput(const std::string& name) const {
435
  auto* var = InputVar(name);
436 437 438 439
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
440
  auto* var = OutputVar(name);
441 442 443
  return var != nullptr;
}

X
Xin Pan 已提交
444
const Variable* ExecutionContext::InputVar(const std::string& name) const {
445 446
  LogVarUsageIfUnusedVarCheckEnabled(name);

X
Xin Pan 已提交
447 448 449
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) return nullptr;

450 451 452 453 454
  PADDLE_ENFORCE_LE(
      it->second.size(), 1UL,
      platform::errors::AlreadyExists(
          "Operator %s's input %s should contain only one variable.",
          op_.Type(), name));
X
Xin Pan 已提交
455 456 457
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
458
Variable* ExecutionContext::OutputVar(const std::string& name) const {
X
Xin Pan 已提交
459 460 461 462 463 464 465 466 467
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) return nullptr;

  PADDLE_ENFORCE_LE(it->second.size(), 1UL,
                    "Operator %s's output %s should contain only one variable.",
                    op_.Type(), name);
  return it->second.empty() ? nullptr : it->second[0];
}

468
template <>
469
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
470
  return Input<LoDTensor>(name);
471 472 473
}

template <>
474
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
475
    const std::string& name) const {
476 477
  LogVarUsageIfUnusedVarCheckEnabled(name);

H
hong 已提交
478 479
  auto vars = MultiInputVar(name);
  if (vars.size() == 0) {
X
Xin Pan 已提交
480 481 482 483 484
    return {};
  }
  std::vector<const Tensor*> res;
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
H
hong 已提交
485
                 [&](const Variable* var) -> const Tensor* {
X
Xin Pan 已提交
486 487 488 489
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "should be LoDTensor, but the received type is %s",
S
sneaxiy 已提交
490
                       ToTypeName(var->Type()));
X
Xin Pan 已提交
491 492 493 494 495
                   return &(var->Get<LoDTensor>());
                 });
  return res;
}

496
template <>
497
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
498
  return Output<LoDTensor>(name);
499 500 501
}

template <>
502
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
503
    const std::string& name) const {
H
hong 已提交
504 505 506
  auto vars = MultiOutputVar(name);

  if (vars.size() == 0) {
507 508
    return {};
  }
509
  std::vector<Tensor*> res;
510 511 512 513 514
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                 [&](Variable* var) -> Tensor* {
                   return var == nullptr ? nullptr
                                         : var->GetMutable<LoDTensor>();
515
                 });
516 517 518
  return res;
}

Y
Yu Yang 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

534 535
class RuntimeInferShapeContext : public InferShapeContext {
 public:
536
  RuntimeInferShapeContext(const OperatorBase& op, const RuntimeContext& ctx)
G
Gabor Buella 已提交
537
      : op_(op), ctx_(ctx) {}
538 539

  bool HasInput(const std::string& name) const override {
540
    // has only one input
X
Xin Pan 已提交
541
    const auto& ins = ctx_.inputs;
542 543
    auto it = ins.find(name);
    if (it == ins.end()) {
544 545
      return false;
    }
546
    const auto& in = it->second;
X
Xin Pan 已提交
547
    if (in.size() == 0) return false;
T
tensor-tang 已提交
548
    PADDLE_ENFORCE_EQ(in.size(), 1UL,
F
fengjiayi 已提交
549
                      "Input %s should not have more than one inputs", name);
X
Xin Pan 已提交
550
    return in[0] != nullptr;
551 552 553
  }

  bool HasOutput(const std::string& name) const override {
554
    // has only one output
X
Xin Pan 已提交
555
    const auto& outs = ctx_.outputs;
556 557
    auto it = outs.find(name);
    if (it == outs.end()) {
558 559
      return false;
    }
560
    const auto& out = it->second;
X
Xin Pan 已提交
561
    if (out.size() == 0) {
562 563
      return false;
    }
T
tensor-tang 已提交
564 565
    PADDLE_ENFORCE_EQ(out.size(), 1UL,
                      "Output %s should not have more than one outputs", name);
X
Xin Pan 已提交
566
    return out[0] != nullptr;
567 568 569
  }

  bool HasInputs(const std::string& name) const override {
X
Xin Pan 已提交
570 571
    const auto& ins = ctx_.inputs;
    auto it = ins.find(name);
X
fix  
Xin Pan 已提交
572
    if (it == ins.end() || it->second.empty()) {
573 574
      return false;
    }
X
Xin Pan 已提交
575 576
    for (auto& input : it->second) {
      if (input == nullptr) {
577 578 579 580 581 582 583
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
X
Xin Pan 已提交
584 585
    const auto& outs = ctx_.outputs;
    auto it = outs.find(name);
X
fix  
Xin Pan 已提交
586
    if (it == outs.end() || it->second.empty()) {
587 588
      return false;
    }
X
Xin Pan 已提交
589 590
    for (auto& output : it->second) {
      if (output == nullptr) {
591 592 593 594 595 596 597 598
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

H
hong 已提交
599
  std::vector<std::string> Inputs(const std::string& name) const override {
600 601 602
    return op_.Inputs(name);
  }

H
hong 已提交
603
  std::vector<std::string> Outputs(const std::string& name) const override {
604 605 606
    return op_.Outputs(name);
  }

607 608
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
X
Xin Pan 已提交
609 610 611 612 613 614 615 616 617
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE(in_it != ctx_.inputs.end() && in_it->second.size() > i,
                   "Inputs %s should have %llu argument", in, i);
    PADDLE_ENFORCE(out_it != ctx_.outputs.end() && out_it->second.size() > j,
                   "Outputs %s should have %llu argument", out, j);

    Variable* in_var = in_it->second[i];
    Variable* out_var = out_it->second[j];
618 619

    PADDLE_ENFORCE(in_var->Type() == out_var->Type(),
X
fix  
Xin Pan 已提交
620
                   "The type of %s and %s is not the same.", in, out);
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
      PADDLE_THROW(
          "Currently, the input type of ShareDim only can be LoDTensor "
          "or SelectedRows.");
    }
  }

H
hong 已提交
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
  void ShareAllLoD(const std::string& in,
                   const std::string& out) const override {
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(),
                      platform::errors::NotFound(
                          "Input [%s] found error in Op [%s]", in, op_.Type()));
    PADDLE_ENFORCE_NE(
        out_it, ctx_.outputs.end(),
        platform::errors::NotFound("Output [%s] found error in Op [%s]", out,
                                   op_.Type()));

    auto& in_var_list = in_it->second;
    auto& out_var_list = out_it->second;

    PADDLE_ENFORCE_EQ(
        in_var_list.size(), out_var_list.size(),
        platform::errors::PreconditionNotMet(
T
tianshuo78520a 已提交
657
            "Op [%s]: Input var size should be equal with output var size",
H
hong 已提交
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
            op_.Type()));

    auto& out_var_names = op_.Outputs(out);

    for (size_t i = 0; i < in_var_list.size(); ++i) {
      if (out_var_names[i] == framework::kEmptyVarName) {
        continue;
      }

      Variable* in_var = in_var_list[i];
      if (!in_var->IsType<LoDTensor>()) return;
      Variable* out_var = out_var_list[i];
      PADDLE_ENFORCE_EQ(out_var->IsType<LoDTensor>(), true,
                        platform::errors::PreconditionNotMet(
                            "The %d-th output of Output(%s) must be LoDTensor.",
                            i, out_var_names[i]));
      auto& in_tensor = in_var->Get<LoDTensor>();
      auto* out_tensor = out_var->GetMutable<LoDTensor>();
      out_tensor->set_lod(in_tensor.lod());
#ifdef PADDLE_WITH_MKLDNN
      if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
        out_tensor->set_layout(in_tensor.layout());
    }
  }

Q
Qiao Longfei 已提交
684 685
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
686 687 688 689 690 691 692 693
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE(in_it != ctx_.inputs.end() && in_it->second.size() > i,
                   "Inputs %s should have %llu argument", in, i);
    PADDLE_ENFORCE(out_it != ctx_.outputs.end() && out_it->second.size() > j,
                   "Outputs %s should have %llu argument", out, j);

    Variable* in_var = in_it->second.at(i);
Q
Qiao Longfei 已提交
694
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
695
    Variable* out_var = out_it->second.at(j);
Q
Qiao Longfei 已提交
696 697
    PADDLE_ENFORCE(out_var->IsType<LoDTensor>(),
                   "The %d-th output of Output(%s) must be LoDTensor.", j, out);
698
    auto& in_tensor = in_var->Get<LoDTensor>();
Q
Qiao Longfei 已提交
699 700
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
701

M
mozga-intel 已提交
702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
721 722
  }

723
  int32_t GetLoDLevel(const std::string& in, size_t i = 0) const override {
724
    PADDLE_THROW(
725
        "GetLoDLevel is only used in compile time. The calculation of "
726 727 728 729
        "output's actual lod is different among operators so that should be "
        "set in the runtime kernel.");
  }

730 731
  void SetLoDLevel(const std::string& out, int32_t lod_level,
                   size_t j = 0) const override {
732
    PADDLE_THROW(
733
        "SetLoDLevel is only used in compile time. The calculation of "
734 735
        "output's actual lod is different among operators so that should be "
        "set in the runtime kernel.");
C
chengduo 已提交
736 737
  }

738 739
  bool IsRuntime() const override { return true; }

740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
  // TODO(paddle-dev): Can this be template?
  std::vector<InferShapeVarPtr> GetInputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = InputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

  std::vector<InferShapeVarPtr> GetOutputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = OutputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

X
Xin Pan 已提交
759 760 761 762 763 764 765 766 767 768 769 770 771
  DDim GetInputDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    PADDLE_ENFORCE_EQ(vars.size(), 1UL,
                      "Input(%s) should hold one element, but now it holds %d",
                      name, vars.size());
    return this->GetDim(vars[0]);
  }

  std::vector<DDim> GetInputsDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    return GetDims(vars);
  }

X
Xin Pan 已提交
772 773 774 775 776 777 778 779 780 781
  std::vector<proto::VarType::Type> GetInputsVarType(
      const std::string& name) const override {
    return GetVarTypes(InputVars(name));
  }

  std::vector<proto::VarType::Type> GetOutputsVarType(
      const std::string& name) const override {
    return GetVarTypes(OutputVars(name));
  }

X
Xin Pan 已提交
782 783 784 785 786 787 788 789 790 791 792 793 794 795
  void SetOutputDim(const std::string& name, const DDim& dim) override {
    auto& vars = OutputVars(name);
    PADDLE_ENFORCE_EQ(vars.size(), 1UL,
                      "Output(%s) should hold one element, but now it holds %d",
                      name, vars.size());
    SetDim(vars[0], dim);
  }

  void SetOutputsDim(const std::string& name,
                     const std::vector<DDim>& dims) override {
    auto& vars = OutputVars(name);
    SetDims(vars, dims);
  }

796
 protected:
X
Xin Pan 已提交
797
  DDim GetDim(Variable* var) const {
F
fengjiayi 已提交
798
    PADDLE_ENFORCE_NOT_NULL(var);
799 800 801 802 803
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
F
fengjiayi 已提交
804
      PADDLE_THROW(
X
Xin Pan 已提交
805
          "Only LoDTensor/SelectedRows support 'GetDim', but Variables "
F
fengjiayi 已提交
806
          "type_id is %s.",
S
sneaxiy 已提交
807
          ToTypeName(var->Type()));
F
fengjiayi 已提交
808 809 810
    }
  }

X
Xin Pan 已提交
811 812 813 814 815 816 817 818
  std::vector<DDim> GetDims(const std::vector<Variable*>& vars) const {
    std::vector<DDim> ret;
    ret.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(ret),
                   [this](Variable* var) { return this->GetDim(var); });
    return ret;
  }

F
fengjiayi 已提交
819
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
Y
Yu Yang 已提交
820
    PADDLE_THROW("Only compile time support this method");
821 822
  }

X
Xin Pan 已提交
823
  void SetDim(Variable* var, const DDim& dim) {
824 825 826 827 828
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
X
Xin Pan 已提交
829
      PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
830
                   ToTypeName(var->Type()));
X
Xin Pan 已提交
831 832 833 834 835 836 837 838 839 840 841 842
    }
  }

  void SetDims(const std::vector<Variable*>& vars,
               const std::vector<DDim>& dims) {
    size_t length = vars.size();
    PADDLE_ENFORCE_EQ(length, dims.size());
    for (size_t i = 0; i < length; ++i) {
      if (vars[i] == nullptr) {
        continue;
      }
      SetDim(vars[i], dims[i]);
843 844 845
    }
  }

F
fengjiayi 已提交
846 847
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
Y
Yu Yang 已提交
848
    PADDLE_THROW("Only compile time support this method");
F
fengjiayi 已提交
849 850
  }

X
Xin Pan 已提交
851 852 853 854 855 856 857 858 859 860 861
  std::vector<proto::VarType::Type> GetVarTypes(
      const std::vector<Variable*>& vars) const {
    std::vector<proto::VarType::Type> retv;
    retv.resize(vars.size());
    std::transform(vars.begin(), vars.end(), retv.begin(),
                   std::bind(std::mem_fn(&RuntimeInferShapeContext::GetVarType),
                             this, std::placeholders::_1));
    return retv;
  }

  proto::VarType::Type GetVarType(Variable* var) const {
862 863 864
    return ToVarType(var->Type());
  }

865 866 867 868 869 870 871 872 873 874 875 876 877 878
 private:
  const std::vector<Variable*>& InputVars(const std::string& name) const {
    auto it = ctx_.inputs.find(name);
    PADDLE_ENFORCE(it != ctx_.inputs.end(),
                   "Operator %s does not have the input %s.", op_.Type(), name);
    return it->second;
  }

  const std::vector<Variable*>& OutputVars(const std::string& name) const {
    auto it = ctx_.outputs.find(name);
    PADDLE_ENFORCE(it != ctx_.outputs.end(),
                   "Operator %s does not have the outputs %s.", op_.Type(),
                   name);
    return it->second;
F
fengjiayi 已提交
879 880
  }

881
  const OperatorBase& op_;
X
Xin Pan 已提交
882
  const RuntimeContext& ctx_;
883 884
};

885 886
static void CheckTensorNANOrInf(const std::string& op_type,
                                const std::string& name,
C
chengduoZH 已提交
887 888 889 890
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
Y
Yu Yang 已提交
891 892
  if (tensor.type() != proto::VarType::FP32 &&
      tensor.type() != proto::VarType::FP64) {
C
chengduoZH 已提交
893 894 895
    return;
  }
  PADDLE_ENFORCE(!framework::TensorContainsInf(tensor),
896
                 "Operator %s output Tensor %s contains Inf", op_type, name);
C
chengduoZH 已提交
897
  PADDLE_ENFORCE(!framework::TensorContainsNAN(tensor),
898
                 "Operator %s output Tensor %s contains NAN", op_type, name);
C
chengduoZH 已提交
899 900
}

B
baojun-nervana 已提交
901
void OperatorWithKernel::RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
902 903
                                           const platform::Place& place,
                                           const RuntimeContext& ctx) const {
904
  RuntimeInferShapeContext infer_shape_ctx(*this, ctx);
B
baojun-nervana 已提交
905 906 907
  this->InferShape(&infer_shape_ctx);
}

L
luotao1 已提交
908 909
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
L
luotao1 已提交
910 911
  // To reduce the elapsed time of HasAttr, we use bool variable to record the
  // result of HasAttr.
912 913 914
  if (!enable_cache_runtime_context_ && HasAttr(kEnableCacheRuntimeContext))
    enable_cache_runtime_context_ = true;
  if (!all_kernels_must_compute_runtime_shape_ &&
L
luotao1 已提交
915
      HasAttr(kAllKernelsMustComputeRuntimeShape))
916
    all_kernels_must_compute_runtime_shape_ = true;
917
  const Scope* cur_scope = &scope;
918
  if (!enable_cache_runtime_context_) {
L
luotao1 已提交
919 920
    RuntimeContext ctx(Inputs(), Outputs(), scope);
    RunImpl(scope, place, &ctx);
921
    pre_scope_ = cur_scope;
L
luotao1 已提交
922
  } else {
923 924 925 926 927 928
    if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
      std::lock_guard<std::mutex> lock(cache_update_mutex_);
      if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
        runtime_ctx_.reset(new RuntimeContext(Inputs(), Outputs(), scope));
        pre_scope_ = cur_scope;
      }
L
luotao1 已提交
929 930 931 932 933 934 935 936
    }
    RunImpl(scope, place, runtime_ctx_.get());
  }
}

void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place,
                                 RuntimeContext* runtime_ctx) const {
Y
Yu Yang 已提交
937
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
938
  auto* dev_ctx = pool.Get(place);
939

940
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
941
    ChooseKernel(*runtime_ctx, scope, place);
942 943
  }

Y
yuyang18 已提交
944 945
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
946 947
  Scope* transfer_scope = nullptr;
  {
948
    platform::RecordEvent record_event("prepare_data",
949
                                       platform::EventRole::kInnerOp);
950 951 952 953
    if (need_prepare_data_) {
      transfer_scope = PrepareData(scope, *kernel_type_,
                                   &transfered_inplace_vars, runtime_ctx);
    }
954
  }
Y
yuyang18 已提交
955 956 957 958
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

959 960
  if (!(kernel_type_->place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(kernel_type_->place_);
961
  }
Q
QI JUN 已提交
962

963
  if (!all_kernels_must_compute_runtime_shape_) {
964
    platform::RecordEvent record_event("infer_shape",
965
                                       platform::EventRole::kInnerOp);
966
    RuntimeInferShapeContext infer_shape_ctx(*this, *runtime_ctx);
967 968
    this->InferShape(&infer_shape_ctx);
  }
969 970 971 972 973

  if (FLAGS_enable_unused_var_check) {
    GetThreadLocalUsedVarNameSet()->clear();
  }

X
clean  
Xin Pan 已提交
974 975
  // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
  // not Scope. Imperative mode only pass inputs and get outputs.
976
  {
977
    platform::RecordEvent record_event("compute",
978
                                       platform::EventRole::kInnerOp);
979 980
    (*kernel_func_)(
        ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx));
981
  }
D
dzhwinter 已提交
982

Y
yuyang18 已提交
983
  if (!transfered_inplace_vars.empty()) {
T
tianshuo78520a 已提交
984
    // there is inplace variable has been transferred.
Y
yuyang18 已提交
985
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
986
  }
987 988 989 990 991 992 993 994
  if (FLAGS_enable_unused_var_check) {
    // skip op that uses mkldnn because it has different memory reuse strategy.
    // use attr here because some GradMakers (like ActivationGradOpMaker) add
    // input when use_mkldnn=true;
    if (!(HasAttr("use_mkldnn") && Attr<bool>("use_mkldnn"))) {
      CheckUnusedVar(*this, scope);
    }
  }
995

D
dzhwinter 已提交
996
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
997
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
998
    dev_ctx->Wait();
D
dzhwinter 已提交
999
  }
C
chengduoZH 已提交
1000

P
pkpk 已提交
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
  if (FLAGS_fast_check_nan_inf) {
    for (auto& vname : OutputVars(true)) {
      // only check inserted vars,
      // please see executor.py for details of fast_check_nan_inf
      if (vname.rfind("debug_var") == 0) {
        VLOG(3) << "debugging nan/inf in var " << vname;

        auto* var = exec_scope.FindVar(vname);
        if (var == nullptr) continue;
        if (var->IsType<framework::LoDTensor>()) {
          CheckTensorNANOrInf(type_, vname, var->Get<framework::LoDTensor>());
        } else if (var->IsType<framework::SelectedRows>()) {
          CheckTensorNANOrInf(type_, vname,
                              var->Get<framework::SelectedRows>().value());
        }
      }
    }
  }

C
chengduoZH 已提交
1020
  if (FLAGS_check_nan_inf) {
W
WangXi 已提交
1021
    framework::details::CheckOpHasNanOrInf(*this, exec_scope, place);
C
chengduoZH 已提交
1022
  }
1023 1024 1025 1026 1027 1028 1029

  // To solve issue #15032, have a discussion with @Luotao for cpu inference,
  // do not cache transfer scope, hence in this case delete transfer scope
  // after run to avoid memory leak
  if (transfer_scope && !run_by_executor_ && !enable_cache_transfer_scope_) {
    scope.DeleteScope(transfer_scope);
  }
Q
Qiao Longfei 已提交
1030
}
X
Xin Pan 已提交
1031

L
Liu Yiqun 已提交
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
void OperatorWithKernel::ChooseKernel(const RuntimeContext& ctx,
                                      const Scope& scope,
                                      const platform::Place& place) const {
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
  auto* dev_ctx = pool.Get(place);

  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  if (kernels_iter == all_op_kernels.end()) {
    PADDLE_THROW(
        "There are no kernels which are registered in the %s operator.", type_);
  }

  OpKernelMap& kernels = kernels_iter->second;

  auto expected_kernel_key = this->GetExpectedKernelType(
1049
      ExecutionContext(*this, scope, *dev_ctx, ctx));
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
  if (HasAttr("op_device")) {
    if (Attr<std::string>("op_device") == "cpu") {
      expected_kernel_key.place_ = platform::CPUPlace();
    } else if (Attr<std::string>("op_device") == "gpu") {
      // when the Op that only has CPUKernel is assigned to GPU, the CPUKernel
      // will be executed and a warning will be given at the same time.
      if (SupportGPU()) {
        expected_kernel_key.place_ = dev_ctx->GetPlace();
      } else {
        expected_kernel_key.place_ = platform::CPUPlace();
        LOG_FIRST_N(WARNING, 1)
            << "Op(" << type_
            << ") has no CUDA implementation. It will be assigned to CPUPlace.";
      }
    }
  }
L
Liu Yiqun 已提交
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
  VLOG(3) << "expected_kernel_key:" << expected_kernel_key;

  auto kernel_iter = kernels.find(expected_kernel_key);
#ifdef PADDLE_WITH_MKLDNN
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
    VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one";
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
#endif
  if (kernel_iter == kernels.end()) {
    PADDLE_THROW("op %s does not have kernel for %s", type_,
                 KernelTypeToString(expected_kernel_key));
  }

1084 1085 1086 1087 1088
  std::lock_guard<std::mutex> lock(cache_update_mutex_);
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
    kernel_type_.reset(new OpKernelType(expected_kernel_key));
    kernel_func_.reset(new OpKernelFunc(kernel_iter->second));
  }
L
Liu Yiqun 已提交
1089 1090
}

Y
yuyang18 已提交
1091 1092 1093 1094
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
M
minqiyang 已提交
1095
    VLOG(3) << "share inplace var " + var_name + " back to it's original scope";
C
chengduo 已提交
1096 1097 1098
    auto* origin_var = scope.FindVar(var_name);
    PADDLE_ENFORCE_NOT_NULL(origin_var, "The var[%s] should not be nullptr.",
                            var_name);
C
chengduo 已提交
1099
    auto* original_tensor =
C
chengduo 已提交
1100
        GetMutableLoDTensorOrSelectedRowsValueFromVar(origin_var);
C
chengduo 已提交
1101
    auto* var = transfer_scope.FindVar(var_name);
C
chengduo 已提交
1102 1103
    PADDLE_ENFORCE_NOT_NULL(var, "The var[%s] should not be nullptr.",
                            var_name);
C
chengduo 已提交
1104
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1105
    auto original_dims = original_tensor->dims();
Y
yuyang18 已提交
1106
    original_tensor->ShareDataWith(*transformed_tensor);
1107
    original_tensor->Resize(original_dims);
Y
yuyang18 已提交
1108 1109 1110
  }
}

X
Xin Pan 已提交
1111
Scope* OperatorWithKernel::PrepareData(
Y
yuyang18 已提交
1112
    const Scope& scope, const OpKernelType& expected_kernel_key,
X
Xin Pan 已提交
1113 1114
    std::vector<std::string>* transfered_inplace_vars,
    RuntimeContext* ctx) const {
Y
yuyang18 已提交
1115
  Scope* new_scope = nullptr;
S
sneaxiy 已提交
1116

1117
  const std::unordered_set<std::string>* no_buffer_ins = nullptr;
S
sneaxiy 已提交
1118 1119 1120 1121
  if (info_) {
    auto& no_buffer_inferer = info_->NoNeedBufferVarsInferer();
    // Some op may not register NoNeedBufferVarsInferer
    if (no_buffer_inferer) {
1122 1123
      no_buffer_ins = &(no_buffer_inferer(Inputs(), Outputs(), Attrs()));
      if (no_buffer_ins->empty()) no_buffer_ins = nullptr;
S
sneaxiy 已提交
1124 1125 1126
    }
  }

Y
yuyang18 已提交
1127
  for (auto& var_name_item : Inputs()) {
1128 1129
    bool should_skip_input =
        no_buffer_ins && no_buffer_ins->count(var_name_item.first) > 0;
S
sneaxiy 已提交
1130

X
Xin Pan 已提交
1131 1132 1133 1134
    std::vector<Variable*>& input_vars = ctx->inputs[var_name_item.first];

    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      auto& var_name = var_name_item.second[i];
X
Xin Pan 已提交
1135
      auto* var = input_vars[i];
X
Xin Pan 已提交
1136

Y
yuyang18 已提交
1137
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
1138
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
1139 1140 1141
        continue;
      }

C
chengduo 已提交
1142
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180

      // When no_buffer_ins then checking of Tensor::holder_ is
      // not a thread safe. And for infershape scenario checks
      // to be omitted are not really needed
      if (should_skip_input == true) {
#ifdef PADDLE_WITH_MKLDNN
        // Var without buffer may be needed
        // for some situation like InferShape().
        // In this situation We cannot skip Var analysis, as
        // MKL-DNN shape of Var may differ from kNHWC Var
        // In such situation corressponding resized Var
        // has to be created and registered
        if ((tensor_in->layout() == DataLayout::kMKLDNN) &&
            (var->IsType<LoDTensor>() == true) &&
            (expected_kernel_key.data_layout_ != DataLayout::kMKLDNN) &&
            (paddle::platform::get_cur_paddle_data_layout() ==
             DataLayout::kNHWC)) {
          // Mixed execution : MKL-DNN and GPU is not supported!
          if (!new_scope) {
            new_scope = &scope.NewScope();
          }
          auto* trans_var = new_scope->Var(var_name);
          input_vars[i] = trans_var;
          auto out = trans_var->GetMutable<LoDTensor>();
          out->Resize(tensor_in->dims());
          platform::MatchShapeToLayout(out, tensor_in->layout(),
                                       DataLayout::kNHWC);
          VLOG(7) << "Created reshaped dummy input based on MKL-DNN Tensor , "
                     "but kNHWC layout"
                  << var_name_item.first << " in Operator " << type_;
        } else {
          VLOG(7) << "Skip scanning input " << var_name_item.first
                  << " in Operator " << type_;
        }
#endif
        continue;
      }

Y
yuyang18 已提交
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

      auto out_var_names = OutputVars(true);
      if (std::find(out_var_names.begin(), out_var_names.end(), var_name) !=
          out_var_names.end()) {
        transfered_inplace_vars->emplace_back(var_name);
      }

M
minqiyang 已提交
1198 1199
      VLOG(3) << "Transform Variable " << var_name << " from "
              << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
1200

1201 1202 1203
      // In the inference scenerio, the scopes will be reused across the
      // batches, so the `new_scope` here will result in GPU memroy explosion
      // over the  running of operators.
1204
      // We use a thread_local cache to fix that issue, the key in the cache is
1205 1206 1207 1208 1209
      // the combination of the `scope` argument, from_kernel_type,
      // target_kernel_type.
      // Have a discussion with @Superjomn or the inference developers if some
      // changes on this logic for this macro might not tested on the other
      // scenerios.
1210 1211
      // If this op is not called by an Executor or ParallelExecutor, it should
      // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and
1212
      // variables, that behavior a lot different.
1213 1214 1215 1216 1217 1218 1219 1220 1221
      //
      // To solve issue #15032, have a discussion with @Luotao for cpu
      // inference, for all cpu kernels cases without GPU participation, here
      // not do transfer scope caching, and cpu inference performance is not
      // impacted by test.
      enable_cache_transfer_scope_ = false;
      if (!run_by_executor_ &&
          (platform::is_gpu_place(kernel_type_for_var.place_) ||
           platform::is_gpu_place(expected_kernel_key.place_))) {
1222 1223
        new_scope = TryCreateTransferScope(kernel_type_for_var,
                                           expected_kernel_key, &scope);
1224
        enable_cache_transfer_scope_ = true;
1225
      }
1226
      if (!new_scope) {
Y
yuyang18 已提交
1227 1228
        new_scope = &scope.NewScope();
      }
1229 1230 1231 1232
      // For inference, if a gpu model has an op which could only run on CPU,
      // each result of different input will be the same with the first one.
      // The reason is that if a gpu tensor is the input of a cpu kernel,
      // we will create a new cpu tensor in new scope.
1233
      // However, if enable_cache_runtime_context_, we get the cpu tensor each
1234 1235
      // time, not the gpu tensor. Thus, we set pre_scope_ = nullptr
      // to trigger `new RuntimeContext()` in RunImpl().
1236
      if (enable_cache_runtime_context_) {
1237 1238
        pre_scope_ = nullptr;
      }
Y
yuyang18 已提交
1239
      auto* trans_var = new_scope->Var(var_name);
X
fix  
Xin Pan 已提交
1240
      input_vars[i] = trans_var;
Y
yuyang18 已提交
1241
      Tensor out;
Y
yuyang18 已提交
1242
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
1243 1244 1245
      SetTensorToVariable(*var, out, trans_var);
    }
  }
1246 1247 1248 1249 1250 1251 1252 1253 1254
  // If pre_scope = &scope, it means that scope is cached and the op is not in
  // while block. If new_scope = nullptr, it means that for each input of this
  // Op, there is no need to do PrepareData. So PrepareData could be skipped at
  // the rest iterations to save the elapsed time.
  // We do not support skipping PrepareData in while block, because the Op's
  // input may be changed by subsequent Ops, which may cause an error.
  if (pre_scope_ == &scope && new_scope == nullptr) {
    need_prepare_data_ = false;
  }
Y
yuyang18 已提交
1255 1256 1257

  return new_scope;
}
Q
Qiao Longfei 已提交
1258

1259 1260 1261
void OperatorWithKernel::ParseInputDataType(
    const ExecutionContext& ctx, const std::string& name,
    proto::VarType::Type* data_type) const {
1262
  proto::VarType::Type default_data_type =
1263
      static_cast<proto::VarType::Type>(-1);
H
hong 已提交
1264
  const std::vector<Variable*> vars = ctx.MultiInputVar(name);
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
  for (size_t i = 0; i < vars.size(); ++i) {
    const Variable* var = vars[i];
    if (var != nullptr) {
      const Tensor* t = nullptr;
      if (var->IsType<Tensor>()) {
        t = &var->Get<Tensor>();
      } else if (var->IsType<LoDTensor>()) {
        t = &var->Get<LoDTensor>();
      } else if (var->IsType<SelectedRows>()) {
        t = &(var->Get<SelectedRows>().value());
1275 1276 1277 1278 1279 1280 1281
      } else if (var->IsType<LoDTensorArray>()) {
        auto t_arr = var->Get<LoDTensorArray>();
        for (size_t j = 0; j < t_arr.size(); j++) {
          if (t_arr[j].IsInitialized()) {
            t = &(t_arr[j]);
          }
        }
1282 1283
      }
      if (t != nullptr) {
1284 1285 1286 1287 1288
        PADDLE_ENFORCE_EQ(
            t->IsInitialized(), true,
            platform::errors::InvalidArgument(
                "The Tensor in the %s Op's Input Variable %s(%s) is "
                "not initialized.",
H
hong 已提交
1289
                Type(), name, ctx.InputNames(name).at(i)));
1290
        proto::VarType::Type tmp = t->type();
1291
        PADDLE_ENFORCE(
1292
            tmp == *data_type || *data_type == default_data_type,
1293 1294 1295 1296 1297 1298
            platform::errors::InvalidArgument(
                "The DataType of %s Op's duplicable Variable %s must be "
                "consistent. The current variable type is (%s), but the "
                "previous variable type is (%s).",
                Type(), name, DataTypeToString(tmp),
                DataTypeToString(*data_type)));
1299 1300 1301 1302 1303 1304
        *data_type = tmp;
      }
    }
  }
}

1305
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
1306
    const ExecutionContext& ctx) const {
1307 1308 1309
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
H
hong 已提交
1310 1311
  for (auto& input : ctx.InNameList()) {
    ParseInputDataType(ctx, input, &data_type);
Y
Yu Yang 已提交
1312
  }
1313 1314 1315 1316
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
      platform::errors::NotFound(
          "DataType should be indicated by input Variable at %s.", Type()));
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
  return data_type;
}

proto::VarType::Type OperatorWithKernel::IndicateVarDataType(
    const ExecutionContext& ctx, const std::string& name) const {
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
  ParseInputDataType(ctx, name, &data_type);
  PADDLE_ENFORCE_NE(
      data_type, dafault_data_type,
      "The Input Variable(%s) of %s Op used to determine kernel data type "
L
liym27 已提交
1329
      "is empty or not LoDTensor or SelectedRows or LoDTensorArray.",
1330
      name, Type());
1331
  return data_type;
Y
Yu Yang 已提交
1332
}
1333

1334 1335 1336 1337 1338 1339 1340 1341
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
1342 1343
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
1344 1345
}

Q
Qiao Longfei 已提交
1346
}  // namespace framework
L
liaogang 已提交
1347
}  // namespace paddle