operator.cc 41.1 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include <gflags/gflags.h>
#include <glog/logging.h>
17

18
#include <algorithm>
P
peizhilin 已提交
19 20
#include <sstream>
#include <string>
S
sneaxiy 已提交
21
#include <unordered_set>
P
peizhilin 已提交
22
#include <vector>
Y
Yi Wang 已提交
23 24
#include "paddle/fluid/framework/data_transform.h"
#include "paddle/fluid/framework/executor.h"
25
#include "paddle/fluid/framework/lod_tensor.h"
26
#include "paddle/fluid/framework/op_proto_maker.h"
27
#include "paddle/fluid/framework/operator.h"
Y
Yi Wang 已提交
28
#include "paddle/fluid/framework/shape_inference.h"
29
#include "paddle/fluid/framework/transfer_scope_cache.h"
Y
Yi Wang 已提交
30
#include "paddle/fluid/framework/var_type.h"
31
#include "paddle/fluid/platform/profiler.h"
Q
Qiao Longfei 已提交
32

D
dzhwinter 已提交
33
DECLARE_bool(benchmark);
C
chengduoZH 已提交
34 35 36
DEFINE_bool(check_nan_inf, false,
            "Checking whether operator produce NAN/INF or not. It will be "
            "extremely slow so please use this flag wisely.");
Q
Qiao Longfei 已提交
37
DEFINE_int32(inner_op_parallelism, 0, "number of threads for inner op");
P
pkpk 已提交
38 39 40
DEFINE_bool(fast_check_nan_inf, false,
            "Fast checking NAN/INF after each operation. It will be a little"
            "bit slow, much faster than check_nan_inf");
D
dzhwinter 已提交
41

Q
Qiao Longfei 已提交
42 43 44
namespace paddle {
namespace framework {

45 46 47 48 49 50
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
51

Q
qiaolongfei 已提交
52 53
proto::VarType::Type GetDataTypeOfVar(const Variable* var) {
  if (var->IsType<framework::LoDTensor>()) {
Y
Yu Yang 已提交
54
    return var->Get<framework::LoDTensor>().type();
Q
qiaolongfei 已提交
55
  } else if (var->IsType<framework::SelectedRows>()) {
Y
Yu Yang 已提交
56
    return var->Get<framework::SelectedRows>().value().type();
Q
qiaolongfei 已提交
57 58 59 60 61
  } else {
    PADDLE_THROW("Var should be LoDTensor or SelectedRows");
  }
}

62 63
static DDim GetDimsDebug(const Scope& scope, const std::string& name,
                         bool get_actual_dim = false) {
64
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
65 66
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
67 68
  }

M
minqiyang 已提交
69 70
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
71 72 73
    if (UNLIKELY(!tensor.IsInitialized())) {
      return DDim({-1});
    }
M
minqiyang 已提交
74 75 76 77 78 79 80
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
81 82 83 84 85
  } else {
    return DDim({-1});
  }
}

Q
Qiao Longfei 已提交
86 87 88 89 90 91
static bool VarInited(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

D
dzhwinter 已提交
92 93 94 95 96
static std::string GetDtype(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
97

M
minqiyang 已提交
98 99 100
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
101 102
      return "";
    }
Y
Yu Yang 已提交
103
    return DataTypeToString(tensor.type());
M
minqiyang 已提交
104
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
105 106 107 108
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
Y
Yu Yang 已提交
109
      return DataTypeToString(tensor.type());
Q
Qiao Longfei 已提交
110
    }
D
dzhwinter 已提交
111 112 113 114 115
  } else {
    return "";
  }
}

116 117 118 119 120 121
static int GetRowSize(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
122 123
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
124 125 126 127 128
  }

  return -1;
}

129
static LoD GetLoDDebug(const Scope& scope, const std::string& name) {
Q
Qiao Longfei 已提交
130 131 132 133 134 135 136
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
137 138
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
139 140 141
    if (UNLIKELY(!tensor.IsInitialized())) {
      return default_lod;
    }
M
minqiyang 已提交
142
    return tensor.lod();
Q
Qiao Longfei 已提交
143 144 145 146 147
  } else {
    return default_lod;
  }
}

X
Xin Pan 已提交
148 149 150 151 152
RuntimeContext::RuntimeContext(const VariableNameMap& innames,
                               const VariableNameMap& outnames,
                               const Scope& scope) {
  for (auto& var_name_item : innames) {
    std::vector<Variable*>& input_vars = inputs[var_name_item.first];
X
Xin Pan 已提交
153
    input_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
154 155 156 157 158 159
    for (auto& var_name : var_name_item.second) {
      input_vars.push_back(scope.FindVar(var_name));
    }
  }
  for (auto& var_name_item : outnames) {
    std::vector<Variable*>& output_vars = outputs[var_name_item.first];
X
Xin Pan 已提交
160
    output_vars.reserve(var_name_item.second.size());
X
Xin Pan 已提交
161 162 163 164 165 166
    for (auto& var_name : var_name_item.second) {
      output_vars.push_back(scope.FindVar(var_name));
    }
  }
}

167
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
P
peizhilin 已提交
168 169 170
  try {
    VLOG(4) << place << " " << DebugStringEx(&scope);
    if (platform::is_gpu_place(place)) {
171
#ifndef PADDLE_WITH_CUDA
P
peizhilin 已提交
172
      PADDLE_THROW("Cannot run operator on place %s", place);
173
#else
P
peizhilin 已提交
174 175
      auto dev_id = boost::get<platform::CUDAPlace>(place).device;
      platform::SetDeviceId(dev_id);
176
#endif
P
peizhilin 已提交
177
    }
P
peizhilin 已提交
178

P
peizhilin 已提交
179 180 181 182 183
    // The profile has a process-wide mutex, results in serious performance
    // issue
    // in concurrency scenerio. Here use an `if` to fix this issue.
    // Please not remove the `if`, ask @Superjomn if there are any concern.
    if (platform::IsProfileEnabled()) {
184
      platform::RecordEvent record_event(Type());
P
peizhilin 已提交
185 186 187 188
      RunImpl(scope, place);
    } else {
      RunImpl(scope, place);
    }
189

P
peizhilin 已提交
190 191 192
    VLOG(3) << place << " " << DebugStringEx(&scope);
  } catch (platform::EnforceNotMet exception) {
    if (Attrs().count("sub_block") != 0) {
193
      throw std::move(exception);
P
peizhilin 已提交
194
    }
195

P
peizhilin 已提交
196 197 198 199
    auto& callstack = Attr<std::vector<std::string>>(
        OpProtoAndCheckerMaker::OpCreationCallstackAttrName());

    if (callstack.empty()) {
200
      throw std::move(exception);
P
peizhilin 已提交
201 202 203 204 205 206 207 208 209 210
    }
    std::ostringstream sout;
    sout << "Invoke operator " << Type() << " error.\n";
    sout << "Python Callstacks: \n";
    for (auto& line : callstack) {
      sout << line;
    }
    sout << "C++ Callstacks: \n";
    sout << exception.err_str_;
    exception.err_str_ = sout.str();
211
    throw std::move(exception);
P
peizhilin 已提交
212 213
  } catch (...) {
    std::rethrow_exception(std::current_exception());
214
  }
215 216
}

217
bool OperatorBase::HasInputs(const std::string& name) const {
M
minqiyang 已提交
218
  return inputs_.find(name) != inputs_.end();
219 220
}

221
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
222
  auto& ins = Inputs(name);
Y
Yu Yang 已提交
223
  PADDLE_ENFORCE_LE(ins.size(), 1UL,
224 225
                    "Operator %s's input %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
226
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
227 228
}

Y
Yu Yang 已提交
229 230
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
231
  auto it = inputs_.find(name);
232 233
  PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.",
                 type_, name);
Y
Yu Yang 已提交
234
  return it->second;
Y
Yan Chunwei 已提交
235 236
}

237
bool OperatorBase::HasOutputs(const std::string& name) const {
238
  if (outputs_.find(name) != outputs_.end()) {
239 240 241 242 243 244
    return true;
  } else {
    return false;
  }
}

245
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
246
  auto& outs = Outputs(name);
Y
Yu Yang 已提交
247
  PADDLE_ENFORCE_LE(outs.size(), 1UL,
248 249
                    "Operator %s's output %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
250
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
251 252
}

Y
Yu Yang 已提交
253 254
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
255
  auto it = outputs_.find(name);
256 257
  PADDLE_ENFORCE(it != outputs_.end(),
                 "Operator %s does not have an output called %s.", type_, name);
Y
Yu Yang 已提交
258
  return it->second;
Y
Yan Chunwei 已提交
259 260
}

261
std::string OperatorBase::DebugStringEx(const Scope* scope) const {
Q
Qiao Longfei 已提交
262
  std::stringstream ss;
Y
Yu Yang 已提交
263
  ss << "Op(" << type_ << "), inputs:{";
Y
Yu Yang 已提交
264 265
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
Y
Yu Yang 已提交
266 267
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
268 269
      auto var_name = input.second[i];
      ss << var_name;
270
      if (scope) {
Q
Qiao Longfei 已提交
271 272 273 274 275 276 277 278 279
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
          std::string dtype = GetDtype(*scope, var_name);
          ss << ":" << dtype;
280 281
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
282
        }
283
      }
Y
Yu Yang 已提交
284 285 286
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
287
    }
Y
Yu Yang 已提交
288
    ss << "]";
Y
Yu Yang 已提交
289 290
    ++it;
    if (it != inputs_.end()) {
291 292
      ss << ", ";
    }
Q
Qiao Longfei 已提交
293
  }
Y
Yu Yang 已提交
294
  ss << "}, outputs:{";
Y
Yu Yang 已提交
295 296
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
297 298
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
299 300
      auto var_name = output.second[i];
      ss << var_name;
301
      if (scope) {
Q
Qiao Longfei 已提交
302 303 304 305 306 307 308
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
309 310
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
311 312
          ss << "[" << GetDimsDebug(*scope, var_name, true) << "]";
          ss << "(" << GetLoDDebug(*scope, var_name) << ")";
313
        }
314
      }
Y
Yu Yang 已提交
315 316 317
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
318
    }
Y
Yu Yang 已提交
319
    ss << "]";
Y
Yu Yang 已提交
320 321
    ++it;
    if (it != outputs_.end()) {
322 323
      ss << ", ";
    }
Q
Qiao Longfei 已提交
324
  }
Y
Yu Yang 已提交
325
  ss << "}.";
Q
Qiao Longfei 已提交
326 327 328
  return ss.str();
}

Y
Yu Yang 已提交
329
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
330 331
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
332
                           const AttributeMap& attrs)
S
sneaxiy 已提交
333 334 335 336 337 338
    : type_(type),
      inputs_(inputs),
      outputs_(outputs),
      attrs_(attrs),
      // NOTE(zjl): why op_info may be nullptr?
      info_(OpInfoMap::Instance().GetNullable(type)) {
339 340
  GenerateTemporaryNames();
  CheckAllInputOutputSet();
Y
Yu Yang 已提交
341
}
342

Q
qijun 已提交
343 344
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
345
  for (auto& o : inputs_) {
Q
qijun 已提交
346 347 348 349 350 351
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
352 353 354 355 356 357 358 359 360 361
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
S
sneaxiy 已提交
362
  auto& info = Info();
Y
Yu Yang 已提交
363 364

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
365
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
366 367 368 369 370 371 372 373 374
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
375 376
}

377
void OperatorBase::CheckAllInputOutputSet() const {
S
sneaxiy 已提交
378
  if (info_ == nullptr || info_->proto_ == nullptr) return;
379

S
sneaxiy 已提交
380
  for (auto& in : info_->Proto().inputs()) {
381 382 383 384
    if (!in.dispensable()) {
      PADDLE_ENFORCE(inputs_.find(in.name()) != inputs_.end(),
                     "Operator %s's input, %s, is not set", Type(), in.name());
    }
385 386
  }

S
sneaxiy 已提交
387
  for (auto& out : info_->Proto().outputs()) {
388 389 390 391 392
    if (!out.dispensable()) {
      PADDLE_ENFORCE(outputs_.find(out.name()) != outputs_.end(),
                     "Operator %s's output, %s, is not set", Type(),
                     out.name());
    }
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

B
baojun-nervana 已提交
409
static bool VarIsTensor(const Variable& var) {
C
chengduo 已提交
410
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
411 412
}

C
chengduo 已提交
413
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
414 415 416 417
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
418
  } else {
Y
Yang Yang 已提交
419
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
420
                 ToTypeName(var.Type()));
Q
QI JUN 已提交
421 422 423
  }
}

C
chengduo 已提交
424
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
425
  if (var->IsType<LoDTensor>()) {
426
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
427
  } else if (var->IsType<SelectedRows>()) {
428
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
429
  } else {
Y
Yang Yang 已提交
430
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
431
                 ToTypeName(var->Type()));
Q
QI JUN 已提交
432 433 434
  }
}

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
bool ExecutionContext::HasInput(const std::string& name) const {
  if (!op_.HasInputs(name)) {
    return false;
  }
  auto& ins = Inputs(name);
  size_t length = ins.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Input %s should not have more than one inputs", name);
  auto arg = ins[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
  if (!op_.HasOutputs(name)) {
    return false;
  }
  auto& outs = Outputs(name);
  size_t length = outs.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Output %s should not have more than one inputs", name);
  auto arg = outs[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

X
Xin Pan 已提交
467 468 469 470 471 472 473 474 475 476
const Variable* ExecutionContext::InputVar(const std::string& name) const {
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) return nullptr;

  PADDLE_ENFORCE_LE(it->second.size(), 1UL,
                    "Operator %s's input %s should contain only one variable.",
                    op_.Type(), name);
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
477
Variable* ExecutionContext::OutputVar(const std::string& name) const {
X
Xin Pan 已提交
478 479 480 481 482 483 484 485 486
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) return nullptr;

  PADDLE_ENFORCE_LE(it->second.size(), 1UL,
                    "Operator %s's output %s should contain only one variable.",
                    op_.Type(), name);
  return it->second.empty() ? nullptr : it->second[0];
}

487
template <>
488
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
489
  return Input<LoDTensor>(name);
490 491 492
}

template <>
493
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
494
    const std::string& name) const {
X
Xin Pan 已提交
495 496 497 498 499 500 501 502 503 504 505 506 507
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) {
    return {};
  }
  const std::vector<Variable*>& vars = it->second;
  std::vector<const Tensor*> res;
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                 [&](Variable* var) -> const Tensor* {
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "should be LoDTensor, but the received type is %s",
S
sneaxiy 已提交
508
                       ToTypeName(var->Type()));
X
Xin Pan 已提交
509 510 511 512 513
                   return &(var->Get<LoDTensor>());
                 });
  return res;
}

514
template <>
515
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
516
  return Output<LoDTensor>(name);
517 518 519
}

template <>
520
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
521
    const std::string& name) const {
522 523 524 525 526
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) {
    return {};
  }
  const std::vector<Variable*>& vars = it->second;
527
  std::vector<Tensor*> res;
528 529 530 531 532
  res.reserve(vars.size());
  std::transform(vars.begin(), vars.end(), std::back_inserter(res),
                 [&](Variable* var) -> Tensor* {
                   return var == nullptr ? nullptr
                                         : var->GetMutable<LoDTensor>();
533
                 });
534 535 536
  return res;
}

Y
Yu Yang 已提交
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

552 553
class RuntimeInferShapeContext : public InferShapeContext {
 public:
X
Xin Pan 已提交
554 555
  RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope,
                           const RuntimeContext& ctx)
G
Gabor Buella 已提交
556
      : op_(op), ctx_(ctx) {}
557 558

  bool HasInput(const std::string& name) const override {
559
    // has only one input
X
Xin Pan 已提交
560
    const auto& ins = ctx_.inputs;
561 562
    auto it = ins.find(name);
    if (it == ins.end()) {
563 564
      return false;
    }
565
    const auto& in = it->second;
X
Xin Pan 已提交
566
    if (in.size() == 0) return false;
T
tensor-tang 已提交
567
    PADDLE_ENFORCE_EQ(in.size(), 1UL,
F
fengjiayi 已提交
568
                      "Input %s should not have more than one inputs", name);
X
Xin Pan 已提交
569
    return in[0] != nullptr;
570 571 572
  }

  bool HasOutput(const std::string& name) const override {
573
    // has only one output
X
Xin Pan 已提交
574
    const auto& outs = ctx_.outputs;
575 576
    auto it = outs.find(name);
    if (it == outs.end()) {
577 578
      return false;
    }
579
    const auto& out = it->second;
X
Xin Pan 已提交
580
    if (out.size() == 0) {
581 582
      return false;
    }
T
tensor-tang 已提交
583 584
    PADDLE_ENFORCE_EQ(out.size(), 1UL,
                      "Output %s should not have more than one outputs", name);
X
Xin Pan 已提交
585
    return out[0] != nullptr;
586 587 588
  }

  bool HasInputs(const std::string& name) const override {
X
Xin Pan 已提交
589 590
    const auto& ins = ctx_.inputs;
    auto it = ins.find(name);
X
fix  
Xin Pan 已提交
591
    if (it == ins.end() || it->second.empty()) {
592 593
      return false;
    }
X
Xin Pan 已提交
594 595
    for (auto& input : it->second) {
      if (input == nullptr) {
596 597 598 599 600 601 602
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
X
Xin Pan 已提交
603 604
    const auto& outs = ctx_.outputs;
    auto it = outs.find(name);
X
fix  
Xin Pan 已提交
605
    if (it == outs.end() || it->second.empty()) {
606 607
      return false;
    }
X
Xin Pan 已提交
608 609
    for (auto& output : it->second) {
      if (output == nullptr) {
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

  const std::vector<std::string>& Inputs(
      const std::string& name) const override {
    return op_.Inputs(name);
  }

  const std::vector<std::string>& Outputs(
      const std::string& name) const override {
    return op_.Outputs(name);
  }

628 629
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
X
Xin Pan 已提交
630 631 632 633 634 635 636 637 638
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE(in_it != ctx_.inputs.end() && in_it->second.size() > i,
                   "Inputs %s should have %llu argument", in, i);
    PADDLE_ENFORCE(out_it != ctx_.outputs.end() && out_it->second.size() > j,
                   "Outputs %s should have %llu argument", out, j);

    Variable* in_var = in_it->second[i];
    Variable* out_var = out_it->second[j];
639 640

    PADDLE_ENFORCE(in_var->Type() == out_var->Type(),
X
fix  
Xin Pan 已提交
641
                   "The type of %s and %s is not the same.", in, out);
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
      PADDLE_THROW(
          "Currently, the input type of ShareDim only can be LoDTensor "
          "or SelectedRows.");
    }
  }

Q
Qiao Longfei 已提交
660 661
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
662 663 664 665 666 667 668 669
    auto in_it = ctx_.inputs.find(in);
    auto out_it = ctx_.outputs.find(out);
    PADDLE_ENFORCE(in_it != ctx_.inputs.end() && in_it->second.size() > i,
                   "Inputs %s should have %llu argument", in, i);
    PADDLE_ENFORCE(out_it != ctx_.outputs.end() && out_it->second.size() > j,
                   "Outputs %s should have %llu argument", out, j);

    Variable* in_var = in_it->second.at(i);
Q
Qiao Longfei 已提交
670
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
671
    Variable* out_var = out_it->second.at(j);
Q
Qiao Longfei 已提交
672 673 674 675 676
    PADDLE_ENFORCE(out_var->IsType<LoDTensor>(),
                   "The %d-th output of Output(%s) must be LoDTensor.", j, out);
    auto in_tensor = in_var->Get<LoDTensor>();
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
677

M
mozga-intel 已提交
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
697 698
  }

C
chengduo 已提交
699 700 701 702 703
  void DecreaseLoDLevel(const std::string& in, const std::string& out,
                        size_t i = 0, size_t j = 0) const override {
    PADDLE_THROW("DecreaseLoDLevel is only used in compile time.");
  }

704 705
  bool IsRuntime() const override { return true; }

706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
  // TODO(paddle-dev): Can this be template?
  std::vector<InferShapeVarPtr> GetInputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = InputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

  std::vector<InferShapeVarPtr> GetOutputVarPtrs(
      const std::string& name) override {
    const std::vector<Variable*>& vars = OutputVars(name);
    std::vector<InferShapeVarPtr> res;
    res.reserve(vars.size());
    res.insert(res.begin(), vars.begin(), vars.end());
    return res;
  }

X
Xin Pan 已提交
725 726 727 728 729 730 731 732 733 734 735 736 737
  DDim GetInputDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    PADDLE_ENFORCE_EQ(vars.size(), 1UL,
                      "Input(%s) should hold one element, but now it holds %d",
                      name, vars.size());
    return this->GetDim(vars[0]);
  }

  std::vector<DDim> GetInputsDim(const std::string& name) const override {
    const std::vector<Variable*>& vars = InputVars(name);
    return GetDims(vars);
  }

X
Xin Pan 已提交
738 739 740 741 742 743 744 745 746 747
  std::vector<proto::VarType::Type> GetInputsVarType(
      const std::string& name) const override {
    return GetVarTypes(InputVars(name));
  }

  std::vector<proto::VarType::Type> GetOutputsVarType(
      const std::string& name) const override {
    return GetVarTypes(OutputVars(name));
  }

X
Xin Pan 已提交
748 749 750 751 752 753 754 755 756 757 758 759 760 761
  void SetOutputDim(const std::string& name, const DDim& dim) override {
    auto& vars = OutputVars(name);
    PADDLE_ENFORCE_EQ(vars.size(), 1UL,
                      "Output(%s) should hold one element, but now it holds %d",
                      name, vars.size());
    SetDim(vars[0], dim);
  }

  void SetOutputsDim(const std::string& name,
                     const std::vector<DDim>& dims) override {
    auto& vars = OutputVars(name);
    SetDims(vars, dims);
  }

762
 protected:
X
Xin Pan 已提交
763
  DDim GetDim(Variable* var) const {
F
fengjiayi 已提交
764
    PADDLE_ENFORCE_NOT_NULL(var);
765 766 767 768 769
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
F
fengjiayi 已提交
770
      PADDLE_THROW(
X
Xin Pan 已提交
771
          "Only LoDTensor/SelectedRows support 'GetDim', but Variables "
F
fengjiayi 已提交
772
          "type_id is %s.",
S
sneaxiy 已提交
773
          ToTypeName(var->Type()));
F
fengjiayi 已提交
774 775 776
    }
  }

X
Xin Pan 已提交
777 778 779 780 781 782 783 784
  std::vector<DDim> GetDims(const std::vector<Variable*>& vars) const {
    std::vector<DDim> ret;
    ret.reserve(vars.size());
    std::transform(vars.begin(), vars.end(), std::back_inserter(ret),
                   [this](Variable* var) { return this->GetDim(var); });
    return ret;
  }

F
fengjiayi 已提交
785
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
Y
Yu Yang 已提交
786
    PADDLE_THROW("Only compile time support this method");
787 788
  }

X
Xin Pan 已提交
789
  void SetDim(Variable* var, const DDim& dim) {
790 791 792 793 794
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
X
Xin Pan 已提交
795
      PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
796
                   ToTypeName(var->Type()));
X
Xin Pan 已提交
797 798 799 800 801 802 803 804 805 806 807 808
    }
  }

  void SetDims(const std::vector<Variable*>& vars,
               const std::vector<DDim>& dims) {
    size_t length = vars.size();
    PADDLE_ENFORCE_EQ(length, dims.size());
    for (size_t i = 0; i < length; ++i) {
      if (vars[i] == nullptr) {
        continue;
      }
      SetDim(vars[i], dims[i]);
809 810 811
    }
  }

F
fengjiayi 已提交
812 813
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
Y
Yu Yang 已提交
814
    PADDLE_THROW("Only compile time support this method");
F
fengjiayi 已提交
815 816
  }

X
Xin Pan 已提交
817 818 819 820 821 822 823 824 825 826 827
  std::vector<proto::VarType::Type> GetVarTypes(
      const std::vector<Variable*>& vars) const {
    std::vector<proto::VarType::Type> retv;
    retv.resize(vars.size());
    std::transform(vars.begin(), vars.end(), retv.begin(),
                   std::bind(std::mem_fn(&RuntimeInferShapeContext::GetVarType),
                             this, std::placeholders::_1));
    return retv;
  }

  proto::VarType::Type GetVarType(Variable* var) const {
828 829 830
    return ToVarType(var->Type());
  }

831 832 833 834 835 836 837 838 839 840 841 842 843 844
 private:
  const std::vector<Variable*>& InputVars(const std::string& name) const {
    auto it = ctx_.inputs.find(name);
    PADDLE_ENFORCE(it != ctx_.inputs.end(),
                   "Operator %s does not have the input %s.", op_.Type(), name);
    return it->second;
  }

  const std::vector<Variable*>& OutputVars(const std::string& name) const {
    auto it = ctx_.outputs.find(name);
    PADDLE_ENFORCE(it != ctx_.outputs.end(),
                   "Operator %s does not have the outputs %s.", op_.Type(),
                   name);
    return it->second;
F
fengjiayi 已提交
845 846
  }

847
  const OperatorBase& op_;
X
Xin Pan 已提交
848
  const RuntimeContext& ctx_;
849 850
};

851 852
static void CheckTensorNANOrInf(const std::string& op_type,
                                const std::string& name,
C
chengduoZH 已提交
853 854 855 856
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
Y
Yu Yang 已提交
857 858
  if (tensor.type() != proto::VarType::FP32 &&
      tensor.type() != proto::VarType::FP64) {
C
chengduoZH 已提交
859 860 861
    return;
  }
  PADDLE_ENFORCE(!framework::TensorContainsInf(tensor),
862
                 "Operator %s output Tensor %s contains Inf", op_type, name);
C
chengduoZH 已提交
863
  PADDLE_ENFORCE(!framework::TensorContainsNAN(tensor),
864
                 "Operator %s output Tensor %s contains NAN", op_type, name);
C
chengduoZH 已提交
865 866
}

B
baojun-nervana 已提交
867
void OperatorWithKernel::RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
868 869 870
                                           const platform::Place& place,
                                           const RuntimeContext& ctx) const {
  RuntimeInferShapeContext infer_shape_ctx(*this, scope, ctx);
B
baojun-nervana 已提交
871 872 873
  this->InferShape(&infer_shape_ctx);
}

X
polish  
Xin Pan 已提交
874 875 876 877 878 879 880 881 882 883
std::vector<KernelConfig>* OperatorWithKernel::GetKernelConfig(
    const OpKernelType& key) const {
  auto config_iter = kernel_configs_map_.find(key);
  std::vector<KernelConfig>* kernel_configs = nullptr;
  if (config_iter != kernel_configs_map_.end()) {
    kernel_configs = &(config_iter->second);
  }
  return kernel_configs;
}

L
luotao1 已提交
884 885
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
L
luotao1 已提交
886 887
  // To reduce the elapsed time of HasAttr, we use bool variable to record the
  // result of HasAttr.
888 889 890
  if (!enable_cache_runtime_context_ && HasAttr(kEnableCacheRuntimeContext))
    enable_cache_runtime_context_ = true;
  if (!all_kernels_must_compute_runtime_shape_ &&
L
luotao1 已提交
891
      HasAttr(kAllKernelsMustComputeRuntimeShape))
892 893
    all_kernels_must_compute_runtime_shape_ = true;
  if (!enable_cache_runtime_context_) {
L
luotao1 已提交
894 895 896 897
    RuntimeContext ctx(Inputs(), Outputs(), scope);
    RunImpl(scope, place, &ctx);
  } else {
    const Scope* cur_scope = &scope;
898 899 900 901 902 903
    if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
      std::lock_guard<std::mutex> lock(cache_update_mutex_);
      if (runtime_ctx_.get() == nullptr || pre_scope_ != cur_scope) {
        runtime_ctx_.reset(new RuntimeContext(Inputs(), Outputs(), scope));
        pre_scope_ = cur_scope;
      }
L
luotao1 已提交
904 905 906 907 908 909 910 911
    }
    RunImpl(scope, place, runtime_ctx_.get());
  }
}

void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place,
                                 RuntimeContext* runtime_ctx) const {
Y
Yu Yang 已提交
912
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
913
  auto* dev_ctx = pool.Get(place);
914

915
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
916
    ChooseKernel(*runtime_ctx, scope, place);
917 918
  }

L
Liu Yiqun 已提交
919
  std::vector<KernelConfig>* kernel_configs = GetKernelConfig(*kernel_type_);
920

Y
yuyang18 已提交
921 922
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
T
Tao Luo 已提交
923
  auto* transfer_scope =
924
      PrepareData(scope, *kernel_type_, &transfered_inplace_vars, runtime_ctx);
925

Y
yuyang18 已提交
926 927 928 929
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

930 931
  if (!(kernel_type_->place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(kernel_type_->place_);
932
  }
Q
QI JUN 已提交
933

934
  if (!all_kernels_must_compute_runtime_shape_) {
L
luotao1 已提交
935
    RuntimeInferShapeContext infer_shape_ctx(*this, exec_scope, *runtime_ctx);
936 937
    this->InferShape(&infer_shape_ctx);
  }
X
clean  
Xin Pan 已提交
938 939
  // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
  // not Scope. Imperative mode only pass inputs and get outputs.
940 941
  (*kernel_func_)(ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx,
                                   kernel_configs));
D
dzhwinter 已提交
942

Y
yuyang18 已提交
943 944 945
  if (!transfered_inplace_vars.empty()) {
    // there is inplace variable has been transfered.
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
946 947
  }

D
dzhwinter 已提交
948
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
949
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
950
    dev_ctx->Wait();
D
dzhwinter 已提交
951
  }
C
chengduoZH 已提交
952

P
pkpk 已提交
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971
  if (FLAGS_fast_check_nan_inf) {
    for (auto& vname : OutputVars(true)) {
      // only check inserted vars,
      // please see executor.py for details of fast_check_nan_inf
      if (vname.rfind("debug_var") == 0) {
        VLOG(3) << "debugging nan/inf in var " << vname;

        auto* var = exec_scope.FindVar(vname);
        if (var == nullptr) continue;
        if (var->IsType<framework::LoDTensor>()) {
          CheckTensorNANOrInf(type_, vname, var->Get<framework::LoDTensor>());
        } else if (var->IsType<framework::SelectedRows>()) {
          CheckTensorNANOrInf(type_, vname,
                              var->Get<framework::SelectedRows>().value());
        }
      }
    }
  }

C
chengduoZH 已提交
972 973
  if (FLAGS_check_nan_inf) {
    for (auto& vname : OutputVars(true)) {
Y
yuyang18 已提交
974
      auto* var = exec_scope.FindVar(vname);
C
chengduoZH 已提交
975 976
      if (var == nullptr) continue;
      if (var->IsType<framework::LoDTensor>()) {
977
        CheckTensorNANOrInf(type_, vname, var->Get<framework::LoDTensor>());
978
      } else if (var->IsType<framework::SelectedRows>()) {
979 980
        CheckTensorNANOrInf(type_, vname,
                            var->Get<framework::SelectedRows>().value());
C
chengduoZH 已提交
981 982 983
      }
    }
  }
984 985 986 987 988 989 990

  // To solve issue #15032, have a discussion with @Luotao for cpu inference,
  // do not cache transfer scope, hence in this case delete transfer scope
  // after run to avoid memory leak
  if (transfer_scope && !run_by_executor_ && !enable_cache_transfer_scope_) {
    scope.DeleteScope(transfer_scope);
  }
Q
Qiao Longfei 已提交
991
}
X
Xin Pan 已提交
992

L
Liu Yiqun 已提交
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
void OperatorWithKernel::ChooseKernel(const RuntimeContext& ctx,
                                      const Scope& scope,
                                      const platform::Place& place) const {
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
  auto* dev_ctx = pool.Get(place);

  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  if (kernels_iter == all_op_kernels.end()) {
    PADDLE_THROW(
        "There are no kernels which are registered in the %s operator.", type_);
  }

  OpKernelMap& kernels = kernels_iter->second;

  auto expected_kernel_key = this->GetExpectedKernelType(
      ExecutionContext(*this, scope, *dev_ctx, ctx, nullptr));
  VLOG(3) << "expected_kernel_key:" << expected_kernel_key;

  auto kernel_iter = kernels.find(expected_kernel_key);
#ifdef PADDLE_WITH_MKLDNN
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
    VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one";
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
#endif
  if (kernel_iter == kernels.end()) {
    PADDLE_THROW("op %s does not have kernel for %s", type_,
                 KernelTypeToString(expected_kernel_key));
  }

1029 1030 1031 1032 1033
  std::lock_guard<std::mutex> lock(cache_update_mutex_);
  if (kernel_type_.get() == nullptr || kernel_func_.get() == nullptr) {
    kernel_type_.reset(new OpKernelType(expected_kernel_key));
    kernel_func_.reset(new OpKernelFunc(kernel_iter->second));
  }
L
Liu Yiqun 已提交
1034 1035
}

Y
yuyang18 已提交
1036 1037 1038 1039
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
M
minqiyang 已提交
1040
    VLOG(3) << "share inplace var " + var_name + " back to it's original scope";
C
chengduo 已提交
1041 1042 1043
    auto* origin_var = scope.FindVar(var_name);
    PADDLE_ENFORCE_NOT_NULL(origin_var, "The var[%s] should not be nullptr.",
                            var_name);
C
chengduo 已提交
1044
    auto* original_tensor =
C
chengduo 已提交
1045
        GetMutableLoDTensorOrSelectedRowsValueFromVar(origin_var);
C
chengduo 已提交
1046
    auto* var = transfer_scope.FindVar(var_name);
C
chengduo 已提交
1047 1048
    PADDLE_ENFORCE_NOT_NULL(var, "The var[%s] should not be nullptr.",
                            var_name);
C
chengduo 已提交
1049
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
1050 1051 1052 1053
    original_tensor->ShareDataWith(*transformed_tensor);
  }
}

X
Xin Pan 已提交
1054
Scope* OperatorWithKernel::PrepareData(
Y
yuyang18 已提交
1055
    const Scope& scope, const OpKernelType& expected_kernel_key,
X
Xin Pan 已提交
1056 1057
    std::vector<std::string>* transfered_inplace_vars,
    RuntimeContext* ctx) const {
Y
yuyang18 已提交
1058
  Scope* new_scope = nullptr;
S
sneaxiy 已提交
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068

  std::unordered_set<std::string> no_buffer_ins;
  if (info_) {
    auto& no_buffer_inferer = info_->NoNeedBufferVarsInferer();
    // Some op may not register NoNeedBufferVarsInferer
    if (no_buffer_inferer) {
      no_buffer_ins = no_buffer_inferer(Inputs(), Outputs(), Attrs());
    }
  }

Y
yuyang18 已提交
1069
  for (auto& var_name_item : Inputs()) {
S
sneaxiy 已提交
1070 1071 1072 1073 1074
    // NOTE(zjl): STL does not guarantee fast std::unordered_set::count when set
    // is empty. At least STL implemented on my mac does calculate hash code
    // of search key even though the set is empty.
    if (!no_buffer_ins.empty() &&
        no_buffer_ins.count(var_name_item.first) > 0) {
G
gongweibao 已提交
1075
      VLOG(7) << "Skip scanning input " << var_name_item.first
S
sneaxiy 已提交
1076
              << " in Operator " << type_;
S
sneaxiy 已提交
1077 1078 1079
      continue;
    }

X
Xin Pan 已提交
1080 1081 1082 1083
    std::vector<Variable*>& input_vars = ctx->inputs[var_name_item.first];

    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      auto& var_name = var_name_item.second[i];
X
Xin Pan 已提交
1084
      auto* var = input_vars[i];
X
Xin Pan 已提交
1085

Y
yuyang18 已提交
1086
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
1087
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
1088 1089 1090
        continue;
      }

C
chengduo 已提交
1091
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

      auto out_var_names = OutputVars(true);
      if (std::find(out_var_names.begin(), out_var_names.end(), var_name) !=
          out_var_names.end()) {
        transfered_inplace_vars->emplace_back(var_name);
      }

M
minqiyang 已提交
1109 1110
      VLOG(3) << "Transform Variable " << var_name << " from "
              << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
1111

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
      // In the inference scenerio, the scopes will be reused across the
      // batches, so the `new_scope` here will result in GPU memroy explosion
      // over the  running of operators.
      // We use a thread_local cache to fix that issue, the key in the cache is
      // the combination of the `scope` argument, from_kernel_type,
      // target_kernel_type.
      // Have a discussion with @Superjomn or the inference developers if some
      // changes on this logic for this macro might not tested on the other
      // scenerios.
      // If this op is not called by an Executor or ParallelExecutor, it should
      // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and
      // variables, that behavior a lot different.
1124 1125 1126 1127 1128 1129 1130 1131 1132
      //
      // To solve issue #15032, have a discussion with @Luotao for cpu
      // inference, for all cpu kernels cases without GPU participation, here
      // not do transfer scope caching, and cpu inference performance is not
      // impacted by test.
      enable_cache_transfer_scope_ = false;
      if (!run_by_executor_ &&
          (platform::is_gpu_place(kernel_type_for_var.place_) ||
           platform::is_gpu_place(expected_kernel_key.place_))) {
1133 1134
        new_scope = TryCreateTransferScope(kernel_type_for_var,
                                           expected_kernel_key, &scope);
1135
        enable_cache_transfer_scope_ = true;
1136
      }
1137
      if (!new_scope) {
Y
yuyang18 已提交
1138 1139
        new_scope = &scope.NewScope();
      }
1140 1141 1142 1143
      // For inference, if a gpu model has an op which could only run on CPU,
      // each result of different input will be the same with the first one.
      // The reason is that if a gpu tensor is the input of a cpu kernel,
      // we will create a new cpu tensor in new scope.
1144
      // However, if enable_cache_runtime_context_, we get the cpu tensor each
1145 1146 1147
      // time, not the gpu tensor.
      // Thus, we set pre_scope_ = nullptr to trigger `new RuntimeContext()` in
      // RunImpl().
1148
      if (enable_cache_runtime_context_) {
1149 1150
        pre_scope_ = nullptr;
      }
Y
yuyang18 已提交
1151 1152

      auto* trans_var = new_scope->Var(var_name);
X
fix  
Xin Pan 已提交
1153
      input_vars[i] = trans_var;
1154

Y
yuyang18 已提交
1155
      Tensor out;
Y
yuyang18 已提交
1156
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
1157 1158 1159 1160 1161 1162
      SetTensorToVariable(*var, out, trans_var);
    }
  }

  return new_scope;
}
Q
Qiao Longfei 已提交
1163

1164
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
1165
    const ExecutionContext& ctx) const {
1166 1167 1168
  proto::VarType::Type dafault_data_type =
      static_cast<proto::VarType::Type>(-1);
  proto::VarType::Type data_type = dafault_data_type;
Y
Yu Yang 已提交
1169
  for (auto& input : this->inputs_) {
X
Xin Pan 已提交
1170 1171 1172
    const std::vector<const Variable*> vars = ctx.MultiInputVar(input.first);
    for (size_t i = 0; i < vars.size(); ++i) {
      const Variable* var = vars[i];
Y
Yu Yang 已提交
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
      if (var != nullptr) {
        const Tensor* t = nullptr;
        if (var->IsType<Tensor>()) {
          t = &var->Get<Tensor>();
        } else if (var->IsType<LoDTensor>()) {
          t = &var->Get<LoDTensor>();
        } else if (var->IsType<SelectedRows>()) {
          t = &(var->Get<SelectedRows>().value());
        }
        if (t != nullptr) {
1183
          PADDLE_ENFORCE(t->IsInitialized(), "Input %s(%lu) is not initialized",
X
Xin Pan 已提交
1184
                         input.first, i);
1185
          proto::VarType::Type tmp = t->type();
1186
          PADDLE_ENFORCE(
1187
              tmp == data_type || data_type == dafault_data_type,
1188
              "DataType of Paddle Op %s %s must be the same. Get (%s) != (%s)",
1189 1190
              Type(), input.first, DataTypeToString(data_type),
              DataTypeToString(tmp));
Y
Yu Yang 已提交
1191 1192 1193 1194 1195
          data_type = tmp;
        }
      }
    }
  }
1196 1197 1198
  PADDLE_ENFORCE(data_type != dafault_data_type,
                 "DataType should be indicated by input");
  return data_type;
Y
Yu Yang 已提交
1199
}
1200

1201 1202 1203 1204 1205 1206 1207 1208
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
1209 1210
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
1211 1212
}

Q
Qiao Longfei 已提交
1213
}  // namespace framework
L
liaogang 已提交
1214
}  // namespace paddle