operator.cc 32.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include <gflags/gflags.h>
#include <glog/logging.h>
17

18
#include <algorithm>
19

Y
Yi Wang 已提交
20 21
#include "paddle/fluid/framework/data_transform.h"
#include "paddle/fluid/framework/executor.h"
22
#include "paddle/fluid/framework/lod_tensor.h"
23
#include "paddle/fluid/framework/operator.h"
Y
Yi Wang 已提交
24
#include "paddle/fluid/framework/shape_inference.h"
25
#include "paddle/fluid/framework/transfer_scope_cache.h"
Y
Yi Wang 已提交
26
#include "paddle/fluid/framework/var_type.h"
27
#include "paddle/fluid/platform/profiler.h"
Q
Qiao Longfei 已提交
28

D
dzhwinter 已提交
29
DECLARE_bool(benchmark);
C
chengduoZH 已提交
30 31 32
DEFINE_bool(check_nan_inf, false,
            "Checking whether operator produce NAN/INF or not. It will be "
            "extremely slow so please use this flag wisely.");
D
dzhwinter 已提交
33

Q
Qiao Longfei 已提交
34 35 36
namespace paddle {
namespace framework {

37 38 39 40 41 42
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
43

Q
qiaolongfei 已提交
44 45
proto::VarType::Type GetDataTypeOfVar(const Variable* var) {
  if (var->IsType<framework::LoDTensor>()) {
Y
Yu Yang 已提交
46
    return var->Get<framework::LoDTensor>().type();
Q
qiaolongfei 已提交
47
  } else if (var->IsType<framework::SelectedRows>()) {
Y
Yu Yang 已提交
48
    return var->Get<framework::SelectedRows>().value().type();
Q
qiaolongfei 已提交
49 50 51 52 53
  } else {
    PADDLE_THROW("Var should be LoDTensor or SelectedRows");
  }
}

54 55
static DDim GetDims(const Scope& scope, const std::string& name,
                    bool get_actual_dim = false) {
56
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
57 58
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
59 60
  }

M
minqiyang 已提交
61 62
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
M
minqiyang 已提交
63
    if (UNLIKELY(!tensor.IsInitialized())) {
64
      return DDim({-1});
65
    }
M
minqiyang 已提交
66 67 68 69 70 71 72
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
73 74 75 76 77
  } else {
    return DDim({-1});
  }
}

Q
Qiao Longfei 已提交
78 79 80 81 82 83
static bool VarInited(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

D
dzhwinter 已提交
84 85 86 87 88
static std::string GetDtype(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
89

M
minqiyang 已提交
90 91 92
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
93 94
      return "";
    }
Y
Yu Yang 已提交
95
    return DataTypeToString(tensor.type());
M
minqiyang 已提交
96
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
97 98 99 100
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
Y
Yu Yang 已提交
101
      return DataTypeToString(tensor.type());
Q
Qiao Longfei 已提交
102
    }
D
dzhwinter 已提交
103 104 105 106 107
  } else {
    return "";
  }
}

108 109 110 111 112 113
static int GetRowSize(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
114 115
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
116 117 118 119 120
  }

  return -1;
}

Q
Qiao Longfei 已提交
121 122 123 124 125 126 127 128
static LoD GetLoD(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
129 130 131
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
132 133
      return default_lod;
    }
M
minqiyang 已提交
134
    return tensor.lod();
Q
Qiao Longfei 已提交
135 136 137 138 139
  } else {
    return default_lod;
  }
}

X
Xin Pan 已提交
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
RuntimeContext::RuntimeContext(const VariableNameMap& innames,
                               const VariableNameMap& outnames,
                               const Scope& scope) {
  for (auto& var_name_item : innames) {
    std::vector<Variable*>& input_vars = inputs[var_name_item.first];
    for (auto& var_name : var_name_item.second) {
      input_vars.push_back(scope.FindVar(var_name));
    }
  }
  for (auto& var_name_item : outnames) {
    std::vector<Variable*>& output_vars = outputs[var_name_item.first];
    for (auto& var_name : var_name_item.second) {
      output_vars.push_back(scope.FindVar(var_name));
    }
  }
}

157
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
M
minqiyang 已提交
158
  VLOG(4) << place << " " << DebugStringEx(&scope);
159
  if (platform::is_gpu_place(place)) {
160
#ifndef PADDLE_WITH_CUDA
161
    PADDLE_THROW("Cannot run operator on place %s", place);
162
#else
163 164
    auto dev_id = boost::get<platform::CUDAPlace>(place).device;
    platform::SetDeviceId(dev_id);
165 166
#endif
  }
167

P
peizhilin 已提交
168 169 170
  // The profile has a process-wide mutex, results in serious performance issue
  // in concurrency scenerio. Here use an `if` to fix this issue.
  // Please not remove the `if`, ask @Superjomn if there are any concern.
171 172 173 174
  if (platform::IsProfileEnabled()) {
    platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
    platform::RecordEvent record_event(Type(), pool.Get(place));
    RunImpl(scope, place);
P
peizhilin 已提交
175
  } else {
176 177
    RunImpl(scope, place);
  }
M
minqiyang 已提交
178
  VLOG(3) << place << " " << DebugStringEx(&scope);
179 180
}

181 182 183 184 185 186 187 188
bool OperatorBase::HasInputs(const std::string& name) const {
  if (inputs_.find(name) != inputs_.end()) {
    return true;
  } else {
    return false;
  }
}

189
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
190
  auto& ins = Inputs(name);
Y
Yu Yang 已提交
191
  PADDLE_ENFORCE_LE(ins.size(), 1UL,
192 193
                    "Operator %s's input %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
194
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
195 196
}

Y
Yu Yang 已提交
197 198
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
199
  auto it = inputs_.find(name);
200 201
  PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.",
                 type_, name);
Y
Yu Yang 已提交
202
  return it->second;
Y
Yan Chunwei 已提交
203 204
}

205
bool OperatorBase::HasOutputs(const std::string& name) const {
206
  if (outputs_.find(name) != outputs_.end()) {
207 208 209 210 211 212
    return true;
  } else {
    return false;
  }
}

213
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
214
  auto& outs = Outputs(name);
Y
Yu Yang 已提交
215
  PADDLE_ENFORCE_LE(outs.size(), 1UL,
216 217
                    "Operator %s's output %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
218
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
219 220
}

Y
Yu Yang 已提交
221 222
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
223
  auto it = outputs_.find(name);
224 225
  PADDLE_ENFORCE(it != outputs_.end(),
                 "Operator %s does not have an output called %s.", type_, name);
Y
Yu Yang 已提交
226
  return it->second;
Y
Yan Chunwei 已提交
227 228
}

229
std::string OperatorBase::DebugStringEx(const Scope* scope) const {
Q
Qiao Longfei 已提交
230
  std::stringstream ss;
Y
Yu Yang 已提交
231
  ss << "Op(" << type_ << "), inputs:{";
Y
Yu Yang 已提交
232 233
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
Y
Yu Yang 已提交
234 235
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
236 237
      auto var_name = input.second[i];
      ss << var_name;
238
      if (scope) {
Q
Qiao Longfei 已提交
239 240 241 242 243 244 245 246 247 248 249
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
          std::string dtype = GetDtype(*scope, var_name);
          ss << ":" << dtype;
          ss << "[" << GetDims(*scope, var_name, true) << "]";
          ss << "(" << GetLoD(*scope, var_name) << ")";
250
        }
251
      }
Y
Yu Yang 已提交
252 253 254
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
255
    }
Y
Yu Yang 已提交
256
    ss << "]";
Y
Yu Yang 已提交
257 258
    ++it;
    if (it != inputs_.end()) {
259 260
      ss << ", ";
    }
Q
Qiao Longfei 已提交
261
  }
Y
Yu Yang 已提交
262
  ss << "}, outputs:{";
Y
Yu Yang 已提交
263 264
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
265 266
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
267 268
      auto var_name = output.second[i];
      ss << var_name;
269
      if (scope) {
Q
Qiao Longfei 已提交
270 271 272 273 274 275 276
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
277 278
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
Q
Qiao Longfei 已提交
279 280
          ss << "[" << GetDims(*scope, var_name, true) << "]";
          ss << "(" << GetLoD(*scope, var_name) << ")";
281
        }
282
      }
Y
Yu Yang 已提交
283 284 285
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
286
    }
Y
Yu Yang 已提交
287
    ss << "]";
Y
Yu Yang 已提交
288 289
    ++it;
    if (it != outputs_.end()) {
290 291
      ss << ", ";
    }
Q
Qiao Longfei 已提交
292
  }
Y
Yu Yang 已提交
293
  ss << "}.";
Q
Qiao Longfei 已提交
294 295 296
  return ss.str();
}

Y
Yu Yang 已提交
297
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
298 299
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
300 301
                           const AttributeMap& attrs)
    : type_(type), inputs_(inputs), outputs_(outputs), attrs_(attrs) {
302 303
  GenerateTemporaryNames();
  CheckAllInputOutputSet();
Y
Yu Yang 已提交
304
}
305

Q
qijun 已提交
306 307
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
308
  for (auto& o : inputs_) {
Q
qijun 已提交
309 310 311 312 313 314
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
315 316 317 318 319 320 321 322 323 324
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
Y
Yu Yang 已提交
325
  auto& info = OpInfoMap::Instance().Get(Type());
Y
Yu Yang 已提交
326 327

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
328
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
329 330 331 332 333 334 335 336 337
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
338 339
}

340 341 342
void OperatorBase::CheckAllInputOutputSet() const {
  auto& info_map = OpInfoMap::Instance();
  auto* op_info = info_map.GetNullable(Type());
Y
Yu Yang 已提交
343
  if (op_info == nullptr || op_info->proto_ == nullptr) return;
344 345

  for (auto& in : op_info->Proto().inputs()) {
346 347 348 349
    if (!in.dispensable()) {
      PADDLE_ENFORCE(inputs_.find(in.name()) != inputs_.end(),
                     "Operator %s's input, %s, is not set", Type(), in.name());
    }
350 351 352
  }

  for (auto& out : op_info->Proto().outputs()) {
353 354 355 356 357
    if (!out.dispensable()) {
      PADDLE_ENFORCE(outputs_.find(out.name()) != outputs_.end(),
                     "Operator %s's output, %s, is not set", Type(),
                     out.name());
    }
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

B
baojun-nervana 已提交
374
static bool VarIsTensor(const Variable& var) {
C
chengduo 已提交
375
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
376 377
}

C
chengduo 已提交
378
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
379 380 381 382
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
383
  } else {
Y
Yang Yang 已提交
384
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
385
                 ToTypeName(var.Type()));
Q
QI JUN 已提交
386 387 388
  }
}

C
chengduo 已提交
389
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
390
  if (var->IsType<LoDTensor>()) {
391
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
392
  } else if (var->IsType<SelectedRows>()) {
393
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
394
  } else {
Y
Yang Yang 已提交
395
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
396
                 ToTypeName(var->Type()));
Q
QI JUN 已提交
397 398 399
  }
}

400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
bool ExecutionContext::HasInput(const std::string& name) const {
  if (!op_.HasInputs(name)) {
    return false;
  }
  auto& ins = Inputs(name);
  size_t length = ins.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Input %s should not have more than one inputs", name);
  auto arg = ins[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
  if (!op_.HasOutputs(name)) {
    return false;
  }
  auto& outs = Outputs(name);
  size_t length = outs.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Output %s should not have more than one inputs", name);
  auto arg = outs[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

X
Xin Pan 已提交
432 433 434 435 436 437 438 439 440 441
const Variable* ExecutionContext::InputVar(const std::string& name) const {
  auto it = ctx_.inputs.find(name);
  if (it == ctx_.inputs.end()) return nullptr;

  PADDLE_ENFORCE_LE(it->second.size(), 1UL,
                    "Operator %s's input %s should contain only one variable.",
                    op_.Type(), name);
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
442 443 444 445
const Variable* ExecutionContext::LegacyInputVar(
    const std::string& name) const {
  auto ipt = op_.Input(name);
  return ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt);
X
Xin Pan 已提交
446 447
}

X
clean  
Xin Pan 已提交
448
Variable* ExecutionContext::OutputVar(const std::string& name) const {
X
Xin Pan 已提交
449 450 451 452 453 454 455 456 457
  auto it = ctx_.outputs.find(name);
  if (it == ctx_.outputs.end()) return nullptr;

  PADDLE_ENFORCE_LE(it->second.size(), 1UL,
                    "Operator %s's output %s should contain only one variable.",
                    op_.Type(), name);
  return it->second.empty() ? nullptr : it->second[0];
}

X
clean  
Xin Pan 已提交
458 459 460 461 462
Variable* ExecutionContext::LegacyOutputVar(const std::string& name) const {
  auto opt = op_.Output(name);
  return opt == kEmptyVarName ? nullptr : scope_.FindVar(opt);
}

463
template <>
464
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
465
  return Input<LoDTensor>(name);
466 467
}

X
Xin Pan 已提交
468
template <>
X
clean  
Xin Pan 已提交
469
const Tensor* ExecutionContext::LegacyInput<Tensor>(
X
Xin Pan 已提交
470
    const std::string& name) const {
X
clean  
Xin Pan 已提交
471
  return LegacyInput<LoDTensor>(name);
X
Xin Pan 已提交
472 473
}

474
template <>
475
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
476 477 478 479
    const std::string& name) const {
  auto names = op().Inputs(name);
  std::vector<const Tensor*> res;
  res.reserve(names.size());
480
  std::transform(names.begin(), names.end(), std::back_inserter(res),
C
chengduo 已提交
481
                 [&](const std::string& sub_name) -> const Tensor* {
482
                   auto var = scope_.FindVar(sub_name);
C
chengduo 已提交
483 484 485 486
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "%s should be LoDTensor, but the received type is %s",
S
sneaxiy 已提交
487
                       sub_name, ToTypeName(var->Type()));
C
chengduo 已提交
488
                   return &(var->Get<LoDTensor>());
489
                 });
490 491 492 493
  return res;
}

template <>
494
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
495
  return Output<LoDTensor>(name);
496 497
}

X
Xin Pan 已提交
498
template <>
X
clean  
Xin Pan 已提交
499 500
Tensor* ExecutionContext::LegacyOutput<Tensor>(const std::string& name) const {
  return LegacyOutput<LoDTensor>(name);
X
Xin Pan 已提交
501 502
}

503
template <>
504
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
505 506 507 508
    const std::string& name) const {
  auto names = op().Outputs(name);
  std::vector<Tensor*> res;
  res.reserve(names.size());
509
  std::transform(names.begin(), names.end(), std::back_inserter(res),
C
chengduo 已提交
510
                 [&](const std::string& sub_name) -> Tensor* {
511
                   auto var = scope_.FindVar(sub_name);
C
chengduo 已提交
512 513 514 515
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "%s should be LoDTensor, but the received type is %s",
S
sneaxiy 已提交
516
                       sub_name, ToTypeName(var->Type()));
C
chengduo 已提交
517
                   return var->GetMutable<LoDTensor>();
518
                 });
519 520 521
  return res;
}

Y
Yu Yang 已提交
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

537 538
class RuntimeInferShapeContext : public InferShapeContext {
 public:
X
Xin Pan 已提交
539 540 541
  RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope,
                           const RuntimeContext& ctx)
      : op_(op), scope_(scope), ctx_(ctx) {}
542 543

  bool HasInput(const std::string& name) const override {
544
    // has only one input
X
Xin Pan 已提交
545
    const auto& ins = ctx_.inputs;
546 547
    auto it = ins.find(name);
    if (it == ins.end()) {
548 549
      return false;
    }
550
    const auto& in = it->second;
X
Xin Pan 已提交
551
    if (in.size() == 0) return false;
T
tensor-tang 已提交
552
    PADDLE_ENFORCE_EQ(in.size(), 1UL,
F
fengjiayi 已提交
553
                      "Input %s should not have more than one inputs", name);
X
Xin Pan 已提交
554
    return in[0] != nullptr;
555 556 557
  }

  bool HasOutput(const std::string& name) const override {
558 559 560 561
    // has only one output
    const auto& outs = op_.Outputs();
    auto it = outs.find(name);
    if (it == outs.end()) {
562 563
      return false;
    }
564
    const auto& out = it->second;
T
tensor-tang 已提交
565
    if (out.size() == 0 || out[0] == kEmptyVarName) {
566 567
      return false;
    }
T
tensor-tang 已提交
568 569
    PADDLE_ENFORCE_EQ(out.size(), 1UL,
                      "Output %s should not have more than one outputs", name);
570
    return scope_.FindVar(out[0]) != nullptr;
571 572 573
  }

  bool HasInputs(const std::string& name) const override {
574 575 576
    if (!op_.HasInputs(name)) {
      return false;
    }
577 578 579 580 581 582 583 584 585 586 587 588 589
    auto inputs = op_.Inputs(name);
    if (inputs.empty()) {
      return false;
    }
    for (auto& input : inputs) {
      if (scope_.FindVar(input) == nullptr) {
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
590 591 592
    if (!op_.HasOutputs(name)) {
      return false;
    }
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
    auto outputs = op_.Outputs(name);
    if (outputs.empty()) {
      return false;
    }
    for (auto& output : outputs) {
      if (scope_.FindVar(output) == nullptr) {
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

  const std::vector<std::string>& Inputs(
      const std::string& name) const override {
    return op_.Inputs(name);
  }

  const std::vector<std::string>& Outputs(
      const std::string& name) const override {
    return op_.Outputs(name);
  }

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
    PADDLE_ENFORCE_LT(i, Inputs(in).size());
    PADDLE_ENFORCE_LT(j, Outputs(out).size());
    const std::string& input_n = Inputs(in)[i];
    const std::string& output_n = Outputs(out)[j];

    Variable* in_var = scope_.FindVar(input_n);
    Variable* out_var = scope_.FindVar(output_n);
    PADDLE_ENFORCE(in_var->Type() == out_var->Type(),
                   "The type of %s and %s is not the same.", output_n,
                   GetDim(input_n));

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
      PADDLE_THROW(
          "Currently, the input type of ShareDim only can be LoDTensor "
          "or SelectedRows.");
    }
  }

Q
Qiao Longfei 已提交
647 648
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
649 650 651 652 653
    const std::vector<std::string>& inputs = Inputs(in);
    const std::vector<std::string>& outputs = Outputs(out);
    PADDLE_ENFORCE_LT(i, inputs.size());
    PADDLE_ENFORCE_LT(j, outputs.size());
    Variable* in_var = scope_.FindVar(inputs.at(i));
Q
Qiao Longfei 已提交
654
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
655
    Variable* out_var = scope_.FindVar(outputs.at(j));
Q
Qiao Longfei 已提交
656 657 658 659 660
    PADDLE_ENFORCE(out_var->IsType<LoDTensor>(),
                   "The %d-th output of Output(%s) must be LoDTensor.", j, out);
    auto in_tensor = in_var->Get<LoDTensor>();
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
661

M
mozga-intel 已提交
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
681 682
  }

C
chengduo 已提交
683 684 685 686 687
  void DecreaseLoDLevel(const std::string& in, const std::string& out,
                        size_t i = 0, size_t j = 0) const override {
    PADDLE_THROW("DecreaseLoDLevel is only used in compile time.");
  }

688 689 690
  bool IsRuntime() const override { return true; }

 protected:
691 692
  DDim GetDim(const std::string& name) const override {
    Variable* var = scope_.FindVar(name);
F
fengjiayi 已提交
693
    PADDLE_ENFORCE_NOT_NULL(var);
694 695 696 697 698
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
F
fengjiayi 已提交
699 700 701
      PADDLE_THROW(
          "Only LoDTensor/SelectedRows support 'GetDim', but Variable %s's "
          "type_id is %s.",
S
sneaxiy 已提交
702
          name, ToTypeName(var->Type()));
F
fengjiayi 已提交
703 704 705
    }
  }

F
fengjiayi 已提交
706
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
Y
Yu Yang 已提交
707
    PADDLE_THROW("Only compile time support this method");
708 709 710 711 712 713 714 715 716
  }

  void SetDim(const std::string& name, const DDim& dim) override {
    Variable* var = scope_.FindVar(name);
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
Y
Yang Yang 已提交
717
      PADDLE_THROW("Variable %s type_id %s, expect LoDTensor/SelectedRows.",
S
sneaxiy 已提交
718
                   name, ToTypeName(var->Type()));
719 720 721
    }
  }

F
fengjiayi 已提交
722 723
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
Y
Yu Yang 已提交
724
    PADDLE_THROW("Only compile time support this method");
F
fengjiayi 已提交
725 726
  }

727
  proto::VarType::Type GetVarType(const std::string& name) const override {
728 729 730 731
    auto* var = scope_.FindVar(name);
    return ToVarType(var->Type());
  }

F
fengjiayi 已提交
732 733 734 735
  InferShapeVarPtr GetVarPtr(const std::string& name) override {
    return scope_.FindVar(name);
  }

736
 private:
737 738
  const OperatorBase& op_;
  const Scope& scope_;
X
Xin Pan 已提交
739
  const RuntimeContext& ctx_;
740 741
};

C
chengduoZH 已提交
742 743 744 745 746
static void CheckTensorNANOrInf(const std::string& name,
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
Y
Yu Yang 已提交
747 748
  if (tensor.type() != proto::VarType::FP32 &&
      tensor.type() != proto::VarType::FP64) {
C
chengduoZH 已提交
749 750 751 752 753 754 755 756
    return;
  }
  PADDLE_ENFORCE(!framework::TensorContainsInf(tensor),
                 "Tensor %s contains Inf", name);
  PADDLE_ENFORCE(!framework::TensorContainsNAN(tensor),
                 "Tensor %s contains NAN", name);
}

B
baojun-nervana 已提交
757
void OperatorWithKernel::RuntimeInferShape(const Scope& scope,
X
Xin Pan 已提交
758 759 760
                                           const platform::Place& place,
                                           const RuntimeContext& ctx) const {
  RuntimeInferShapeContext infer_shape_ctx(*this, scope, ctx);
B
baojun-nervana 已提交
761 762 763
  this->InferShape(&infer_shape_ctx);
}

764 765
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
X
Xin Pan 已提交
766
  RuntimeContext ctx(Inputs(), Outputs(), scope);
Y
Yu Yang 已提交
767
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
768
  auto* dev_ctx = pool.Get(place);
769

770 771 772 773
  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  if (kernels_iter == all_op_kernels.end()) {
Y
Yu Yang 已提交
774 775
    PADDLE_THROW(
        "There are no kernels which are registered in the %s operator.", type_);
776 777
  }

Q
qiaolongfei 已提交
778 779
  OpKernelMap& kernels = kernels_iter->second;

X
Xin Pan 已提交
780 781
  auto expected_kernel_key = this->GetExpectedKernelType(
      ExecutionContext(*this, scope, *dev_ctx, ctx));
M
minqiyang 已提交
782
  VLOG(3) << "expected_kernel_key:" << expected_kernel_key;
Q
qiaolongfei 已提交
783

784
  auto kernel_iter = kernels.find(expected_kernel_key);
785
#ifdef PADDLE_WITH_MKLDNN
P
Paweł Żelazko 已提交
786
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
787 788
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
M
minqiyang 已提交
789
    VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one";
790 791 792 793 794
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
#endif
795 796 797 798 799
  if (kernel_iter == kernels.end()) {
    PADDLE_THROW("op %s does not have kernel for %s", type_,
                 KernelTypeToString(expected_kernel_key));
  }

Y
yuyang18 已提交
800 801 802
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
  auto* transfer_scope =
X
Xin Pan 已提交
803
      PrepareData(scope, expected_kernel_key, &transfered_inplace_vars, &ctx);
804

Y
yuyang18 已提交
805 806 807 808 809 810
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

  if (!(expected_kernel_key.place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(expected_kernel_key.place_);
811
  }
Q
QI JUN 已提交
812

X
Xin Pan 已提交
813
  RuntimeInferShapeContext infer_shape_ctx(*this, exec_scope, ctx);
X
Xin Pan 已提交
814
  this->InferShape(&infer_shape_ctx);
X
clean  
Xin Pan 已提交
815 816
  // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
  // not Scope. Imperative mode only pass inputs and get outputs.
X
Xin Pan 已提交
817
  kernel_iter->second(ExecutionContext(*this, exec_scope, *dev_ctx, ctx));
D
dzhwinter 已提交
818

Y
yuyang18 已提交
819 820 821
  if (!transfered_inplace_vars.empty()) {
    // there is inplace variable has been transfered.
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
822 823
  }

D
dzhwinter 已提交
824
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
825
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
826
    dev_ctx->Wait();
D
dzhwinter 已提交
827
  }
C
chengduoZH 已提交
828 829 830

  if (FLAGS_check_nan_inf) {
    for (auto& vname : OutputVars(true)) {
Y
yuyang18 已提交
831
      auto* var = exec_scope.FindVar(vname);
C
chengduoZH 已提交
832 833 834
      if (var == nullptr) continue;
      if (var->IsType<framework::LoDTensor>()) {
        CheckTensorNANOrInf(vname, var->Get<framework::LoDTensor>());
835 836
      } else if (var->IsType<framework::SelectedRows>()) {
        CheckTensorNANOrInf(vname, var->Get<framework::SelectedRows>().value());
C
chengduoZH 已提交
837 838 839
      }
    }
  }
Q
Qiao Longfei 已提交
840
}
X
Xin Pan 已提交
841

Y
yuyang18 已提交
842 843 844 845
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
M
minqiyang 已提交
846
    VLOG(3) << "share inplace var " + var_name + " back to it's original scope";
C
chengduo 已提交
847 848
    auto* original_tensor =
        GetMutableLoDTensorOrSelectedRowsValueFromVar(scope.FindVar(var_name));
C
chengduo 已提交
849 850 851
    auto* var = transfer_scope.FindVar(var_name);
    PADDLE_ENFORCE(var != nullptr, "The var[%s] should not be nullptr",
                   var_name);
C
chengduo 已提交
852
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
853 854 855 856
    original_tensor->ShareDataWith(*transformed_tensor);
  }
}

X
Xin Pan 已提交
857
Scope* OperatorWithKernel::PrepareData(
Y
yuyang18 已提交
858
    const Scope& scope, const OpKernelType& expected_kernel_key,
X
Xin Pan 已提交
859 860
    std::vector<std::string>* transfered_inplace_vars,
    RuntimeContext* ctx) const {
Y
yuyang18 已提交
861 862
  Scope* new_scope = nullptr;
  for (auto& var_name_item : Inputs()) {
X
Xin Pan 已提交
863 864 865 866
    std::vector<Variable*>& input_vars = ctx->inputs[var_name_item.first];

    for (size_t i = 0; i < var_name_item.second.size(); ++i) {
      auto& var_name = var_name_item.second[i];
Y
yuyang18 已提交
867
      auto* var = scope.FindVar(var_name);
X
Xin Pan 已提交
868 869
      input_vars[i] = var;

Y
yuyang18 已提交
870
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
871
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
872 873 874
        continue;
      }

C
chengduo 已提交
875
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

      auto out_var_names = OutputVars(true);
      if (std::find(out_var_names.begin(), out_var_names.end(), var_name) !=
          out_var_names.end()) {
        transfered_inplace_vars->emplace_back(var_name);
      }

M
minqiyang 已提交
893 894
      VLOG(3) << "Transform Variable " << var_name << " from "
              << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
895

896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
      // In the inference scenerio, the scopes will be reused across the
      // batches, so the `new_scope` here will result in GPU memroy explosion
      // over the  running of operators.
      // We use a thread_local cache to fix that issue, the key in the cache is
      // the combination of the `scope` argument, from_kernel_type,
      // target_kernel_type.
      // Have a discussion with @Superjomn or the inference developers if some
      // changes on this logic for this macro might not tested on the other
      // scenerios.
      // If this op is not called by an Executor or ParallelExecutor, it should
      // called by a NaiveExecutor, the NaiveExecutor will cache the scopes and
      // variables, that behavior a lot different.
      if (!run_by_executor_) {
        new_scope = TryCreateTransferScope(kernel_type_for_var,
                                           expected_kernel_key, &scope);
911
      }
912
      if (!new_scope) {
Y
yuyang18 已提交
913 914 915 916
        new_scope = &scope.NewScope();
      }

      auto* trans_var = new_scope->Var(var_name);
X
fix  
Xin Pan 已提交
917
      input_vars[i] = trans_var;
918

Y
yuyang18 已提交
919
      Tensor out;
Y
yuyang18 已提交
920
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
921 922 923 924 925 926
      SetTensorToVariable(*var, out, trans_var);
    }
  }

  return new_scope;
}
Q
Qiao Longfei 已提交
927

928
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
929 930 931
    const ExecutionContext& ctx) const {
  auto& scope = ctx.scope();
  int data_type = -1;
932
  std::string last_input_name;
Y
Yu Yang 已提交
933 934 935 936 937 938 939 940 941 942 943 944 945
  for (auto& input : this->inputs_) {
    for (auto& ipt_name : input.second) {
      auto* var = scope.FindVar(ipt_name);
      if (var != nullptr) {
        const Tensor* t = nullptr;
        if (var->IsType<Tensor>()) {
          t = &var->Get<Tensor>();
        } else if (var->IsType<LoDTensor>()) {
          t = &var->Get<LoDTensor>();
        } else if (var->IsType<SelectedRows>()) {
          t = &(var->Get<SelectedRows>().value());
        }
        if (t != nullptr) {
S
fix bug  
sneaxiy 已提交
946 947
          PADDLE_ENFORCE(t->IsInitialized(), "Input %s is not initialized: %s",
                         ipt_name, DebugString());
Y
Yu Yang 已提交
948
          int tmp = static_cast<int>(t->type());
949 950
          PADDLE_ENFORCE(
              tmp == data_type || data_type == -1,
951 952
              "DataType of Paddle Op %s must be the same. Get %s(%d) != %s(%d)",
              Type(), last_input_name, data_type, ipt_name, tmp);
Y
Yu Yang 已提交
953
          data_type = tmp;
954
          last_input_name = ipt_name;
Y
Yu Yang 已提交
955 956 957 958 959
        }
      }
    }
  }
  PADDLE_ENFORCE(data_type != -1, "DataType should be indicated by input");
960
  return static_cast<proto::VarType::Type>(data_type);
Y
Yu Yang 已提交
961
}
962

963 964 965 966 967 968 969 970
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
971 972
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
973 974
}

Q
Qiao Longfei 已提交
975
}  // namespace framework
L
liaogang 已提交
976
}  // namespace paddle