operator.cc 29.7 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Q
Qiao Longfei 已提交
2 3 4 5 6 7 8 9 10 11 12 13

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14

15 16
#include <gflags/gflags.h>
#include <glog/logging.h>
17

18
#include <algorithm>
19

Y
Yi Wang 已提交
20 21
#include "paddle/fluid/framework/data_transform.h"
#include "paddle/fluid/framework/executor.h"
22
#include "paddle/fluid/framework/lod_tensor.h"
23
#include "paddle/fluid/framework/operator.h"
Y
Yi Wang 已提交
24 25
#include "paddle/fluid/framework/shape_inference.h"
#include "paddle/fluid/framework/var_type.h"
26
#include "paddle/fluid/platform/profiler.h"
Q
Qiao Longfei 已提交
27

D
dzhwinter 已提交
28
DECLARE_bool(benchmark);
C
chengduoZH 已提交
29 30 31
DEFINE_bool(check_nan_inf, false,
            "Checking whether operator produce NAN/INF or not. It will be "
            "extremely slow so please use this flag wisely.");
D
dzhwinter 已提交
32

Q
Qiao Longfei 已提交
33 34 35
namespace paddle {
namespace framework {

36 37 38 39 40
// Combine two hash values to a single hash.
inline size_t CombineHash(size_t seed, size_t a) {
  return (seed ^ a) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}

41 42 43 44 45 46
std::vector<std::tuple<platform::Place, LibraryType>> kKernelPriority = {
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kCUDNN),
    std::make_tuple(platform::CUDAPlace(0), LibraryType::kPlain),
    std::make_tuple(platform::CPUPlace(), LibraryType::kMKLDNN),
    std::make_tuple(platform::CPUPlace(), LibraryType::kPlain),
};
D
dzhwinter 已提交
47

Q
qiaolongfei 已提交
48 49 50 51 52 53 54 55 56 57 58
proto::VarType::Type GetDataTypeOfVar(const Variable* var) {
  if (var->IsType<framework::LoDTensor>()) {
    return framework::ToDataType(var->Get<framework::LoDTensor>().type());
  } else if (var->IsType<framework::SelectedRows>()) {
    return framework::ToDataType(
        var->Get<framework::SelectedRows>().value().type());
  } else {
    PADDLE_THROW("Var should be LoDTensor or SelectedRows");
  }
}

59 60
static DDim GetDims(const Scope& scope, const std::string& name,
                    bool get_actual_dim = false) {
61
  Variable* var = scope.FindVar(name);
Q
qiaolongfei 已提交
62 63
  if (var == nullptr) {
    return DDim({-1});
Q
Qiao Longfei 已提交
64 65
  }

M
minqiyang 已提交
66 67
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
M
minqiyang 已提交
68
    if (UNLIKELY(!tensor.IsInitialized())) {
69
      return DDim({-1});
70
    }
M
minqiyang 已提交
71 72 73 74 75 76 77
    return tensor.dims();
  } else if (var->IsType<SelectedRows>()) {
    if (get_actual_dim) {
      return var->Get<SelectedRows>().value().dims();
    } else {
      return var->Get<SelectedRows>().GetCompleteDims();
    }
78 79 80 81 82
  } else {
    return DDim({-1});
  }
}

Q
Qiao Longfei 已提交
83 84 85 86 87 88
static bool VarInited(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) return false;
  return var->IsInitialized();
}

D
dzhwinter 已提交
89 90 91 92 93
static std::string GetDtype(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return "";
  }
94

M
minqiyang 已提交
95 96 97
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
98 99
      return "";
    }
M
minqiyang 已提交
100 101
    return DataTypeToString(ToDataType(tensor.type()));
  } else if (var->IsType<SelectedRows>()) {
Q
Qiao Longfei 已提交
102 103 104 105 106 107
    auto tensor = var->Get<SelectedRows>().value();
    if (UNLIKELY(!tensor.IsInitialized())) {
      return "uninited";
    } else {
      return DataTypeToString(ToDataType(tensor.type()));
    }
D
dzhwinter 已提交
108 109 110 111 112
  } else {
    return "";
  }
}

113 114 115 116 117 118
static int GetRowSize(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  if (var == nullptr) {
    return -1;
  }

M
minqiyang 已提交
119 120
  if (var->IsType<SelectedRows>()) {
    return var->Get<SelectedRows>().rows().size();
121 122 123 124 125
  }

  return -1;
}

Q
Qiao Longfei 已提交
126 127 128 129 130 131 132 133
static LoD GetLoD(const Scope& scope, const std::string& name) {
  Variable* var = scope.FindVar(name);
  auto default_lod = LoD({{}});

  if (var == nullptr) {
    return default_lod;
  }

M
minqiyang 已提交
134 135 136
  if (var->IsType<LoDTensor>()) {
    const LoDTensor& tensor = var->Get<LoDTensor>();
    if (UNLIKELY(!tensor.IsInitialized())) {
137 138
      return default_lod;
    }
M
minqiyang 已提交
139
    return tensor.lod();
Q
Qiao Longfei 已提交
140 141 142 143 144
  } else {
    return default_lod;
  }
}

145
void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
146
  VLOG(40) << place << " " << DebugStringEx(&scope);
147
  if (platform::is_gpu_place(place)) {
148
#ifndef PADDLE_WITH_CUDA
149
    PADDLE_THROW("Cannot run operator on place %s", place);
150
#else
151 152
    auto dev_id = boost::get<platform::CUDAPlace>(place).device;
    platform::SetDeviceId(dev_id);
153 154
#endif
  }
155

P
peizhilin 已提交
156 157 158
  // The profile has a process-wide mutex, results in serious performance issue
  // in concurrency scenerio. Here use an `if` to fix this issue.
  // Please not remove the `if`, ask @Superjomn if there are any concern.
159 160 161 162
  if (platform::IsProfileEnabled()) {
    platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
    platform::RecordEvent record_event(Type(), pool.Get(place));
    RunImpl(scope, place);
P
peizhilin 已提交
163
  } else {
164 165
    RunImpl(scope, place);
  }
166
  VLOG(30) << place << " " << DebugStringEx(&scope);
167 168
}

169 170 171 172 173 174 175 176
bool OperatorBase::HasInputs(const std::string& name) const {
  if (inputs_.find(name) != inputs_.end()) {
    return true;
  } else {
    return false;
  }
}

177
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
178
  auto& ins = Inputs(name);
Y
Yu Yang 已提交
179
  PADDLE_ENFORCE_LE(ins.size(), 1UL,
180 181
                    "Operator %s's input %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
182
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
183 184
}

Y
Yu Yang 已提交
185 186
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
187
  auto it = inputs_.find(name);
188 189
  PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.",
                 type_, name);
Y
Yu Yang 已提交
190
  return it->second;
Y
Yan Chunwei 已提交
191 192
}

193
bool OperatorBase::HasOutputs(const std::string& name) const {
194
  if (outputs_.find(name) != outputs_.end()) {
195 196 197 198 199 200
    return true;
  } else {
    return false;
  }
}

201
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
202
  auto& outs = Outputs(name);
Y
Yu Yang 已提交
203
  PADDLE_ENFORCE_LE(outs.size(), 1UL,
204 205
                    "Operator %s's output %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
206
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
207 208
}

Y
Yu Yang 已提交
209 210
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
211
  auto it = outputs_.find(name);
212 213
  PADDLE_ENFORCE(it != outputs_.end(),
                 "Operator %s does not have an output called %s.", type_, name);
Y
Yu Yang 已提交
214
  return it->second;
Y
Yan Chunwei 已提交
215 216
}

217
std::string OperatorBase::DebugStringEx(const Scope* scope) const {
Q
Qiao Longfei 已提交
218
  std::stringstream ss;
Y
Yu Yang 已提交
219
  ss << "Op(" << type_ << "), inputs:{";
Y
Yu Yang 已提交
220 221
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
Y
Yu Yang 已提交
222 223
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
Q
Qiao Longfei 已提交
224 225
      auto var_name = input.second[i];
      ss << var_name;
226
      if (scope) {
Q
Qiao Longfei 已提交
227 228 229 230 231 232 233 234 235 236 237
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, var_name);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
          std::string dtype = GetDtype(*scope, var_name);
          ss << ":" << dtype;
          ss << "[" << GetDims(*scope, var_name, true) << "]";
          ss << "(" << GetLoD(*scope, var_name) << ")";
238
        }
239
      }
Y
Yu Yang 已提交
240 241 242
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
243
    }
Y
Yu Yang 已提交
244
    ss << "]";
Y
Yu Yang 已提交
245 246
    ++it;
    if (it != inputs_.end()) {
247 248
      ss << ", ";
    }
Q
Qiao Longfei 已提交
249
  }
Y
Yu Yang 已提交
250
  ss << "}, outputs:{";
Y
Yu Yang 已提交
251 252
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
253 254
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
Q
Qiao Longfei 已提交
255 256
      auto var_name = output.second[i];
      ss << var_name;
257
      if (scope) {
Q
Qiao Longfei 已提交
258 259 260 261 262 263 264
        if (!VarInited(*scope, var_name)) {
          ss << "[uninited]";
        } else {
          int row_size = GetRowSize(*scope, output.second[i]);
          if (row_size >= 0) {
            ss << "[row_size=" << row_size << "]";
          }
C
chengduo 已提交
265 266
          std::string dtype = GetDtype(*scope, output.second[i]);
          ss << ":" << dtype;
Q
Qiao Longfei 已提交
267 268
          ss << "[" << GetDims(*scope, var_name, true) << "]";
          ss << "(" << GetLoD(*scope, var_name) << ")";
269
        }
270
      }
Y
Yu Yang 已提交
271 272 273
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
274
    }
Y
Yu Yang 已提交
275
    ss << "]";
Y
Yu Yang 已提交
276 277
    ++it;
    if (it != outputs_.end()) {
278 279
      ss << ", ";
    }
Q
Qiao Longfei 已提交
280
  }
Y
Yu Yang 已提交
281
  ss << "}.";
Q
Qiao Longfei 已提交
282 283 284
  return ss.str();
}

Y
Yu Yang 已提交
285
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
286 287
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
288 289
                           const AttributeMap& attrs)
    : type_(type), inputs_(inputs), outputs_(outputs), attrs_(attrs) {
290 291
  GenerateTemporaryNames();
  CheckAllInputOutputSet();
Y
Yu Yang 已提交
292
}
293

Q
qijun 已提交
294 295
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
296
  for (auto& o : inputs_) {
Q
qijun 已提交
297 298 299 300 301 302
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
303 304 305 306 307 308 309 310 311 312
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
Y
Yu Yang 已提交
313
  auto& info = OpInfoMap::Instance().Get(Type());
Y
Yu Yang 已提交
314 315

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
316
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
317 318 319 320 321 322 323 324 325
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
326 327
}

328 329 330
void OperatorBase::CheckAllInputOutputSet() const {
  auto& info_map = OpInfoMap::Instance();
  auto* op_info = info_map.GetNullable(Type());
Y
Yu Yang 已提交
331
  if (op_info == nullptr || op_info->proto_ == nullptr) return;
332 333

  for (auto& in : op_info->Proto().inputs()) {
334 335 336 337
    if (!in.dispensable()) {
      PADDLE_ENFORCE(inputs_.find(in.name()) != inputs_.end(),
                     "Operator %s's input, %s, is not set", Type(), in.name());
    }
338 339 340
  }

  for (auto& out : op_info->Proto().outputs()) {
341 342 343 344 345
    if (!out.dispensable()) {
      PADDLE_ENFORCE(outputs_.find(out.name()) != outputs_.end(),
                     "Operator %s's output, %s, is not set", Type(),
                     out.name());
    }
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

C
chengduo 已提交
362 363
static bool VarIsTensor(const Variable& var) {
  return var.IsType<LoDTensor>() || var.IsType<SelectedRows>();
364 365
}

C
chengduo 已提交
366
const Tensor* GetLoDTensorOrSelectedRowsValueFromVar(const Variable& var) {
C
chengduo 已提交
367 368 369 370
  if (var.IsType<LoDTensor>()) {
    return static_cast<const Tensor*>(&(var.Get<LoDTensor>()));
  } else if (var.IsType<SelectedRows>()) {
    return &(var.Get<SelectedRows>().value());
Q
QI JUN 已提交
371
  } else {
Y
Yang Yang 已提交
372
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
C
chengduo 已提交
373
                 var.Type().name());
Q
QI JUN 已提交
374 375 376
  }
}

C
chengduo 已提交
377
Tensor* GetMutableLoDTensorOrSelectedRowsValueFromVar(Variable* var) {
Q
QI JUN 已提交
378
  if (var->IsType<LoDTensor>()) {
379
    return var->GetMutable<LoDTensor>();
Q
QI JUN 已提交
380
  } else if (var->IsType<SelectedRows>()) {
381
    return var->GetMutable<SelectedRows>()->mutable_value();
Q
QI JUN 已提交
382
  } else {
Y
Yang Yang 已提交
383 384
    PADDLE_THROW("Variable type_id %s, expect LoDTensor/SelectedRows.",
                 var->Type().name());
Q
QI JUN 已提交
385 386 387
  }
}

388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
bool ExecutionContext::HasInput(const std::string& name) const {
  if (!op_.HasInputs(name)) {
    return false;
  }
  auto& ins = Inputs(name);
  size_t length = ins.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Input %s should not have more than one inputs", name);
  auto arg = ins[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

bool ExecutionContext::HasOutput(const std::string& name) const {
  if (!op_.HasOutputs(name)) {
    return false;
  }
  auto& outs = Outputs(name);
  size_t length = outs.size();
  if (length == 0) {
    return false;
  }
  PADDLE_ENFORCE_EQ(length, 1UL,
                    "Output %s should not have more than one inputs", name);
  auto arg = outs[0];
  auto* var = arg == kEmptyVarName ? nullptr : scope_.FindVar(arg);
  return var != nullptr;
}

420
template <>
421
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
C
chengduo 已提交
422
  return Input<LoDTensor>(name);
423 424 425
}

template <>
426
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
427 428 429 430
    const std::string& name) const {
  auto names = op().Inputs(name);
  std::vector<const Tensor*> res;
  res.reserve(names.size());
431
  std::transform(names.begin(), names.end(), std::back_inserter(res),
C
chengduo 已提交
432
                 [&](const std::string& sub_name) -> const Tensor* {
433
                   auto var = scope_.FindVar(sub_name);
C
chengduo 已提交
434 435 436 437 438 439
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "%s should be LoDTensor, but the received type is %s",
                       sub_name, var->Type().name());
                   return &(var->Get<LoDTensor>());
440
                 });
441 442 443 444
  return res;
}

template <>
445
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
C
chengduo 已提交
446
  return Output<LoDTensor>(name);
447 448 449
}

template <>
450
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
451 452 453 454
    const std::string& name) const {
  auto names = op().Outputs(name);
  std::vector<Tensor*> res;
  res.reserve(names.size());
455
  std::transform(names.begin(), names.end(), std::back_inserter(res),
C
chengduo 已提交
456
                 [&](const std::string& sub_name) -> Tensor* {
457
                   auto var = scope_.FindVar(sub_name);
C
chengduo 已提交
458 459 460 461 462 463
                   if (var == nullptr) return nullptr;
                   PADDLE_ENFORCE(
                       var->IsType<LoDTensor>(),
                       "%s should be LoDTensor, but the received type is %s",
                       sub_name, var->Type().name());
                   return var->GetMutable<LoDTensor>();
464
                 });
465 466 467
  return res;
}

Y
Yu Yang 已提交
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

483 484 485 486 487 488
class RuntimeInferShapeContext : public InferShapeContext {
 public:
  RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope)
      : op_(op), scope_(scope) {}

  bool HasInput(const std::string& name) const override {
489 490 491 492
    // has only one input
    const auto& ins = op_.Inputs();
    auto it = ins.find(name);
    if (it == ins.end()) {
493 494
      return false;
    }
495
    const auto& in = it->second;
T
tensor-tang 已提交
496
    if (in.size() == 0 || in[0] == kEmptyVarName) {
497 498
      return false;
    }
T
tensor-tang 已提交
499
    PADDLE_ENFORCE_EQ(in.size(), 1UL,
F
fengjiayi 已提交
500
                      "Input %s should not have more than one inputs", name);
501
    return scope_.FindVar(in[0]) != nullptr;
502 503 504
  }

  bool HasOutput(const std::string& name) const override {
505 506 507 508
    // has only one output
    const auto& outs = op_.Outputs();
    auto it = outs.find(name);
    if (it == outs.end()) {
509 510
      return false;
    }
511
    const auto& out = it->second;
T
tensor-tang 已提交
512
    if (out.size() == 0 || out[0] == kEmptyVarName) {
513 514
      return false;
    }
T
tensor-tang 已提交
515 516
    PADDLE_ENFORCE_EQ(out.size(), 1UL,
                      "Output %s should not have more than one outputs", name);
517
    return scope_.FindVar(out[0]) != nullptr;
518 519 520
  }

  bool HasInputs(const std::string& name) const override {
521 522 523
    if (!op_.HasInputs(name)) {
      return false;
    }
524 525 526 527 528 529 530 531 532 533 534 535 536
    auto inputs = op_.Inputs(name);
    if (inputs.empty()) {
      return false;
    }
    for (auto& input : inputs) {
      if (scope_.FindVar(input) == nullptr) {
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
537 538 539
    if (!op_.HasOutputs(name)) {
      return false;
    }
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
    auto outputs = op_.Outputs(name);
    if (outputs.empty()) {
      return false;
    }
    for (auto& output : outputs) {
      if (scope_.FindVar(output) == nullptr) {
        return false;
      }
    }
    return true;
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

  const std::vector<std::string>& Inputs(
      const std::string& name) const override {
    return op_.Inputs(name);
  }

  const std::vector<std::string>& Outputs(
      const std::string& name) const override {
    return op_.Outputs(name);
  }

564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
  void ShareDim(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) override {
    PADDLE_ENFORCE_LT(i, Inputs(in).size());
    PADDLE_ENFORCE_LT(j, Outputs(out).size());
    const std::string& input_n = Inputs(in)[i];
    const std::string& output_n = Outputs(out)[j];

    Variable* in_var = scope_.FindVar(input_n);
    Variable* out_var = scope_.FindVar(output_n);
    PADDLE_ENFORCE(in_var->Type() == out_var->Type(),
                   "The type of %s and %s is not the same.", output_n,
                   GetDim(input_n));

    if (in_var->IsType<framework::SelectedRows>()) {
      auto& in_sele_rows = in_var->Get<framework::SelectedRows>();
      auto out_sele_rows = out_var->GetMutable<framework::SelectedRows>();
      out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims());
      out_sele_rows->set_rows(in_sele_rows.rows());
      out_sele_rows->set_height(in_sele_rows.height());
    } else if (in_var->IsType<framework::LoDTensor>()) {
      auto& in_lod_tensor = in_var->Get<framework::LoDTensor>();
      auto* out_lod_tensor = out_var->GetMutable<framework::LoDTensor>();
      out_lod_tensor->Resize(in_lod_tensor.dims());
    } else {
      PADDLE_THROW(
          "Currently, the input type of ShareDim only can be LoDTensor "
          "or SelectedRows.");
    }
  }

Q
Qiao Longfei 已提交
594 595
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
X
Xin Pan 已提交
596 597 598 599 600
    const std::vector<std::string>& inputs = Inputs(in);
    const std::vector<std::string>& outputs = Outputs(out);
    PADDLE_ENFORCE_LT(i, inputs.size());
    PADDLE_ENFORCE_LT(j, outputs.size());
    Variable* in_var = scope_.FindVar(inputs.at(i));
Q
Qiao Longfei 已提交
601
    if (!in_var->IsType<LoDTensor>()) return;
X
Xin Pan 已提交
602
    Variable* out_var = scope_.FindVar(outputs.at(j));
Q
Qiao Longfei 已提交
603 604 605 606 607
    PADDLE_ENFORCE(out_var->IsType<LoDTensor>(),
                   "The %d-th output of Output(%s) must be LoDTensor.", j, out);
    auto in_tensor = in_var->Get<LoDTensor>();
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
D
dzhwinter 已提交
608

M
mozga-intel 已提交
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
// TODO(dzhwinter) : reuse ShareLoD in most operators.
// Need to call ShareLayout explicitly in sequence related ops.
// Shall we have a better method to shared info between in/out Tensor?
#ifdef PADDLE_WITH_MKLDNN
    // Fix me: ugly workaround below
    // Correct solution:
    //    set_layout() should NOT be called here (i.e. ShareLoD). Instead,
    //    layout of output tensor should be set "manually" in Compute()
    //    of each OPKernel. The reason layout should NOT be shared between
    //    input and output "automatically" (now by InferShape()->ShareLoD())
    //    is that layout transform may occur after InferShape().
    // Workaround:
    //    Skip set_layout() when input layout is kMKLDNN
    //    This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN
    //    OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called
    //    in Compute()
    if (in_tensor.layout() != DataLayout::kMKLDNN)
#endif
      out_tensor->set_layout(in_tensor.layout());
D
dzhwinter 已提交
628 629
  }

630 631 632
  bool IsRuntime() const override { return true; }

 protected:
633 634
  DDim GetDim(const std::string& name) const override {
    Variable* var = scope_.FindVar(name);
F
fengjiayi 已提交
635
    PADDLE_ENFORCE_NOT_NULL(var);
636 637 638 639 640
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
F
fengjiayi 已提交
641 642 643 644 645 646 647
      PADDLE_THROW(
          "Only LoDTensor/SelectedRows support 'GetDim', but Variable %s's "
          "type_id is %s.",
          name, var->Type().name());
    }
  }

F
fengjiayi 已提交
648
  std::vector<DDim> GetRepeatedDims(const std::string& name) const override {
Y
Yu Yang 已提交
649
    PADDLE_THROW("Only compile time support this method");
650 651 652 653 654 655 656 657 658
  }

  void SetDim(const std::string& name, const DDim& dim) override {
    Variable* var = scope_.FindVar(name);
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
Y
Yang Yang 已提交
659 660
      PADDLE_THROW("Variable %s type_id %s, expect LoDTensor/SelectedRows.",
                   name, var->Type().name());
661 662 663
    }
  }

F
fengjiayi 已提交
664 665
  void SetRepeatedDims(const std::string& name,
                       const std::vector<DDim>& dims) override {
Y
Yu Yang 已提交
666
    PADDLE_THROW("Only compile time support this method");
F
fengjiayi 已提交
667 668
  }

669
  proto::VarType::Type GetVarType(const std::string& name) const override {
670 671 672 673
    auto* var = scope_.FindVar(name);
    return ToVarType(var->Type());
  }

F
fengjiayi 已提交
674 675 676 677
  InferShapeVarPtr GetVarPtr(const std::string& name) override {
    return scope_.FindVar(name);
  }

678
 private:
679 680 681 682
  const OperatorBase& op_;
  const Scope& scope_;
};

C
chengduoZH 已提交
683 684 685 686 687
static void CheckTensorNANOrInf(const std::string& name,
                                const framework::Tensor& tensor) {
  if (tensor.memory_size() == 0) {
    return;
  }
S
sneaxiy 已提交
688
  if (!IsType<float>(tensor.type()) && !IsType<double>(tensor.type())) {
C
chengduoZH 已提交
689 690 691 692 693 694 695 696
    return;
  }
  PADDLE_ENFORCE(!framework::TensorContainsInf(tensor),
                 "Tensor %s contains Inf", name);
  PADDLE_ENFORCE(!framework::TensorContainsNAN(tensor),
                 "Tensor %s contains NAN", name);
}

697 698
void OperatorWithKernel::RunImpl(const Scope& scope,
                                 const platform::Place& place) const {
699 700
  RuntimeInferShapeContext infer_shape_ctx(*this, scope);
  this->InferShape(&infer_shape_ctx);
Y
Yu Yang 已提交
701
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
702
  auto* dev_ctx = pool.Get(place);
703

704 705 706 707
  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  if (kernels_iter == all_op_kernels.end()) {
Y
Yu Yang 已提交
708 709
    PADDLE_THROW(
        "There are no kernels which are registered in the %s operator.", type_);
710 711
  }

Q
qiaolongfei 已提交
712 713
  OpKernelMap& kernels = kernels_iter->second;

714 715
  // TODO(dzhwinter) : kernel fallback mechanism will be added when all the
  // transform functions are ready.
Q
qiaolongfei 已提交
716

717 718 719 720
  // for (auto& candidate : kKernelPriority) {
  //   Do selection
  // }

Y
yuyang18 已提交
721 722
  auto expected_kernel_key =
      this->GetExpectedKernelType(ExecutionContext(*this, scope, *dev_ctx));
723
  VLOG(30) << "expected_kernel_key:" << expected_kernel_key;
Q
qiaolongfei 已提交
724

725
  auto kernel_iter = kernels.find(expected_kernel_key);
726
#ifdef PADDLE_WITH_MKLDNN
P
Paweł Żelazko 已提交
727
  // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set
728 729
  if (kernel_iter == kernels.end() &&
      expected_kernel_key.library_type_ == LibraryType::kMKLDNN) {
730
    VLOG(30) << "missing MKLDNN kernel: fallbacking to PLAIN one";
731 732 733 734 735
    expected_kernel_key.library_type_ = LibraryType::kPlain;
    expected_kernel_key.data_layout_ = DataLayout::kAnyLayout;
    kernel_iter = kernels.find(expected_kernel_key);
  }
#endif
736 737 738 739 740
  if (kernel_iter == kernels.end()) {
    PADDLE_THROW("op %s does not have kernel for %s", type_,
                 KernelTypeToString(expected_kernel_key));
  }

Y
yuyang18 已提交
741 742 743 744
  // do data transformScope &transfer_scope;
  std::vector<std::string> transfered_inplace_vars;
  auto* transfer_scope =
      TryTransferData(scope, expected_kernel_key, &transfered_inplace_vars);
745

Y
yuyang18 已提交
746 747 748 749 750 751
  // exec scope is the scope that kernel actually executed on.
  const Scope& exec_scope =
      (transfer_scope == nullptr ? scope : *transfer_scope);

  if (!(expected_kernel_key.place_ == dev_ctx->GetPlace())) {
    dev_ctx = pool.Get(expected_kernel_key.place_);
752
  }
Q
QI JUN 已提交
753

Y
yuyang18 已提交
754
  kernel_iter->second(ExecutionContext(*this, exec_scope, *dev_ctx));
D
dzhwinter 已提交
755

Y
yuyang18 已提交
756 757 758
  if (!transfered_inplace_vars.empty()) {
    // there is inplace variable has been transfered.
    TransferInplaceVarsBack(scope, transfered_inplace_vars, *transfer_scope);
759 760
  }

D
dzhwinter 已提交
761
  /*For profiling/benchmark only*/
D
dzhwinter 已提交
762
  if (FLAGS_benchmark) {
Y
yuyang18 已提交
763
    dev_ctx->Wait();
D
dzhwinter 已提交
764
  }
C
chengduoZH 已提交
765 766 767

  if (FLAGS_check_nan_inf) {
    for (auto& vname : OutputVars(true)) {
Y
yuyang18 已提交
768
      auto* var = exec_scope.FindVar(vname);
C
chengduoZH 已提交
769 770 771
      if (var == nullptr) continue;
      if (var->IsType<framework::LoDTensor>()) {
        CheckTensorNANOrInf(vname, var->Get<framework::LoDTensor>());
772 773
      } else if (var->IsType<framework::SelectedRows>()) {
        CheckTensorNANOrInf(vname, var->Get<framework::SelectedRows>().value());
C
chengduoZH 已提交
774 775 776
      }
    }
  }
Q
Qiao Longfei 已提交
777
}
Y
yuyang18 已提交
778 779 780 781
void OperatorWithKernel::TransferInplaceVarsBack(
    const Scope& scope, const std::vector<std::string>& inplace_vars,
    const Scope& transfer_scope) const {
  for (auto& var_name : inplace_vars) {
782 783
    VLOG(30) << "share inplace var " + var_name +
                    " back to it's original scope";
C
chengduo 已提交
784 785
    auto* original_tensor =
        GetMutableLoDTensorOrSelectedRowsValueFromVar(scope.FindVar(var_name));
C
chengduo 已提交
786 787 788
    auto* var = transfer_scope.FindVar(var_name);
    PADDLE_ENFORCE(var != nullptr, "The var[%s] should not be nullptr",
                   var_name);
C
chengduo 已提交
789
    auto* transformed_tensor = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
790 791 792 793 794 795 796
    original_tensor->ShareDataWith(*transformed_tensor);
  }
}

Scope* OperatorWithKernel::TryTransferData(
    const Scope& scope, const OpKernelType& expected_kernel_key,
    std::vector<std::string>* transfered_inplace_vars) const {
797 798 799 800 801 802 803 804 805 806 807
// In the inference scenerio, the scopes will be reused across the batches, so
// the `new_scope` here will result in GPU memroy explosion over the running of
// operators.
// We use a thread_local cache to fix that issue, the key in the cache is the
// combination of the `scope` argument, from_kernel_type, target_kernel_type.
// Have a discussion with @Superjomn or the inference developers if some changes
// on this logic for this macro might not tested on the other scenerios.
#ifdef PADDLE_ON_INFERENCE
  thread_local std::unordered_map<size_t, Scope*> infer_transfer_scope_cache;
#endif

Y
yuyang18 已提交
808 809 810 811 812
  Scope* new_scope = nullptr;
  for (auto& var_name_item : Inputs()) {
    for (auto& var_name : var_name_item.second) {
      auto* var = scope.FindVar(var_name);
      // Only tensor can be tranfer to another device.
C
chengduo 已提交
813
      if (var == nullptr || !VarIsTensor(*var)) {
Y
yuyang18 已提交
814 815 816
        continue;
      }

C
chengduo 已提交
817
      auto* tensor_in = GetLoDTensorOrSelectedRowsValueFromVar(*var);
Y
yuyang18 已提交
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
      if (!tensor_in->IsInitialized()) {
        continue;
      }

      auto kernel_type_for_var = GetKernelTypeForVar(
          var_name_item.first, *tensor_in, expected_kernel_key);

      if (!NeedTransform(kernel_type_for_var, expected_kernel_key)) {
        continue;
      }

      auto out_var_names = OutputVars(true);
      if (std::find(out_var_names.begin(), out_var_names.end(), var_name) !=
          out_var_names.end()) {
        transfered_inplace_vars->emplace_back(var_name);
      }

835 836
      VLOG(30) << "Transform Variable " << var_name << " from "
               << kernel_type_for_var << " to " << expected_kernel_key;
Y
yuyang18 已提交
837

838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
#ifdef PADDLE_ON_INFERENCE
      size_t infer_cache_key =
          CombineHash(OpKernelType::Hash()(kernel_type_for_var),
                      OpKernelType::Hash()(expected_kernel_key));
      infer_cache_key =
          CombineHash(infer_cache_key, std::hash<const Scope*>()(&scope));

      auto it = infer_transfer_scope_cache.find(infer_cache_key);
      if (it != infer_transfer_scope_cache.end()) {
        new_scope = infer_transfer_scope_cache[infer_cache_key];
      } else {
        new_scope = &scope.NewScope();
        infer_transfer_scope_cache[infer_cache_key] = new_scope;
      }
#endif

Y
yuyang18 已提交
854 855 856 857 858
      if (new_scope == nullptr) {
        new_scope = &scope.NewScope();
      }

      auto* trans_var = new_scope->Var(var_name);
859

Y
yuyang18 已提交
860
      Tensor out;
Y
yuyang18 已提交
861
      TransformData(expected_kernel_key, kernel_type_for_var, *tensor_in, &out);
Y
yuyang18 已提交
862 863 864 865 866 867
      SetTensorToVariable(*var, out, trans_var);
    }
  }

  return new_scope;
}
Q
Qiao Longfei 已提交
868

869
proto::VarType::Type OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
870 871 872
    const ExecutionContext& ctx) const {
  auto& scope = ctx.scope();
  int data_type = -1;
873
  std::string last_input_name;
Y
Yu Yang 已提交
874 875 876 877 878 879 880 881 882 883 884 885 886 887
  for (auto& input : this->inputs_) {
    for (auto& ipt_name : input.second) {
      auto* var = scope.FindVar(ipt_name);
      if (var != nullptr) {
        const Tensor* t = nullptr;
        if (var->IsType<Tensor>()) {
          t = &var->Get<Tensor>();
        } else if (var->IsType<LoDTensor>()) {
          t = &var->Get<LoDTensor>();
        } else if (var->IsType<SelectedRows>()) {
          t = &(var->Get<SelectedRows>().value());
        }
        if (t != nullptr) {
          int tmp = static_cast<int>(ToDataType(t->type()));
888 889
          PADDLE_ENFORCE(
              tmp == data_type || data_type == -1,
890 891
              "DataType of Paddle Op %s must be the same. Get %s(%d) != %s(%d)",
              Type(), last_input_name, data_type, ipt_name, tmp);
Y
Yu Yang 已提交
892
          data_type = tmp;
893
          last_input_name = ipt_name;
Y
Yu Yang 已提交
894 895 896 897 898
        }
      }
    }
  }
  PADDLE_ENFORCE(data_type != -1, "DataType should be indicated by input");
899
  return static_cast<proto::VarType::Type>(data_type);
Y
Yu Yang 已提交
900
}
901

902 903 904 905 906 907 908 909
OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const ExecutionContext& ctx) const {
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
}

OpKernelType OperatorWithKernel::GetKernelTypeForVar(
    const std::string& var_name, const Tensor& tensor,
    const OpKernelType& expected_kernel_type) const {
M
mozga-intel 已提交
910 911
  return OpKernelType(expected_kernel_type.data_type_, tensor.place(),
                      tensor.layout());
912 913
}

Q
Qiao Longfei 已提交
914
}  // namespace framework
L
liaogang 已提交
915
}  // namespace paddle