operator.cc 16.4 KB
Newer Older
Q
Qiao Longfei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

15
#include <algorithm>
T
tensor-tang 已提交
16
#include <atomic>
D
dzhwinter 已提交
17

18
#include "paddle/framework/data_transform.h"
D
dzhwinter 已提交
19
#include "paddle/framework/executor.h"
20
#include "paddle/framework/lod_tensor_array.h"
D
dzhwinter 已提交
21
#include "paddle/framework/operator.h"
22
#include "paddle/framework/shape_inference.h"
23
#include "paddle/framework/var_type.h"
Q
Qiao Longfei 已提交
24 25 26 27

namespace paddle {
namespace framework {

28
std::string OperatorBase::Input(const std::string& name) const {
Y
Yu Yang 已提交
29
  auto& ins = Inputs(name);
Y
Yu Yang 已提交
30
  PADDLE_ENFORCE_LE(ins.size(), 1UL,
31 32
                    "Operator %s's input %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
33
  return ins.empty() ? kEmptyVarName : ins[0];
Y
Yan Chunwei 已提交
34 35
}

Y
Yu Yang 已提交
36 37
const std::vector<std::string>& OperatorBase::Inputs(
    const std::string& name) const {
Y
Yu Yang 已提交
38
  auto it = inputs_.find(name);
39 40
  PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.",
                 type_, name);
Y
Yu Yang 已提交
41
  return it->second;
Y
Yan Chunwei 已提交
42 43
}

44
std::string OperatorBase::Output(const std::string& name) const {
Y
Yu Yang 已提交
45
  auto& outs = Outputs(name);
Y
Yu Yang 已提交
46
  PADDLE_ENFORCE_LE(outs.size(), 1UL,
47 48
                    "Operator %s's output %s should contain only one variable.",
                    type_, name);
Y
Yu Yang 已提交
49
  return outs.empty() ? kEmptyVarName : outs[0];
Y
Yan Chunwei 已提交
50 51
}

Y
Yu Yang 已提交
52 53
const std::vector<std::string>& OperatorBase::Outputs(
    const std::string& name) const {
Y
Yu Yang 已提交
54
  auto it = outputs_.find(name);
55 56
  PADDLE_ENFORCE(it != outputs_.end(),
                 "Operator %s does not have an output called %s.", type_, name);
Y
Yu Yang 已提交
57
  return it->second;
Y
Yan Chunwei 已提交
58 59
}

Q
Qiao Longfei 已提交
60 61
std::string OperatorBase::DebugString() const {
  std::stringstream ss;
Y
Yu Yang 已提交
62
  ss << "Op(" << type_ << "), inputs:{";
Y
Yu Yang 已提交
63 64
  for (auto it = inputs_.begin(); it != inputs_.end();) {
    auto& input = *it;
Y
Yu Yang 已提交
65 66 67 68 69 70
    ss << input.first << "[";
    for (size_t i = 0; i < input.second.size(); ++i) {
      ss << input.second[i];
      if (i != input.second.size() - 1) {
        ss << ", ";
      }
71
    }
Y
Yu Yang 已提交
72
    ss << "]";
Y
Yu Yang 已提交
73 74
    ++it;
    if (it != inputs_.end()) {
75 76
      ss << ", ";
    }
Q
Qiao Longfei 已提交
77
  }
Y
Yu Yang 已提交
78
  ss << "}, outputs:{";
Y
Yu Yang 已提交
79 80
  for (auto it = outputs_.begin(); it != outputs_.end();) {
    auto& output = *it;
Y
Yu Yang 已提交
81 82 83 84 85 86
    ss << output.first << "[";
    for (size_t i = 0; i < output.second.size(); ++i) {
      ss << output.second[i];
      if (i != output.second.size() - 1) {
        ss << ", ";
      }
87
    }
Y
Yu Yang 已提交
88
    ss << "]";
Y
Yu Yang 已提交
89 90
    ++it;
    if (it != outputs_.end()) {
91 92
      ss << ", ";
    }
Q
Qiao Longfei 已提交
93
  }
Y
Yu Yang 已提交
94
  ss << "}.";
Q
Qiao Longfei 已提交
95 96 97
  return ss.str();
}

D
dongzhihong 已提交
98 99
void OperatorBase::Rename(const std::string& old_name,
                          const std::string& new_name) {
Y
Yu Yang 已提交
100 101 102 103 104 105 106
  for (auto& input : inputs_) {
    std::replace(input.second.begin(), input.second.end(), old_name, new_name);
  }
  for (auto& output : outputs_) {
    std::replace(output.second.begin(), output.second.end(), old_name,
                 new_name);
  }
D
dongzhihong 已提交
107 108
}

Y
Yu Yang 已提交
109
OperatorBase::OperatorBase(const std::string& type,
Y
Yu Yang 已提交
110 111
                           const VariableNameMap& inputs,
                           const VariableNameMap& outputs,
Y
Yu Yang 已提交
112 113
                           const AttributeMap& attrs)
    : type_(type), inputs_(inputs), outputs_(outputs), attrs_(attrs) {
114 115
  GenerateTemporaryNames();
  CheckAllInputOutputSet();
Y
Yu Yang 已提交
116
}
117

Q
qijun 已提交
118 119
std::vector<std::string> OperatorBase::InputVars() const {
  std::vector<std::string> ret_val;
Y
Yu Yang 已提交
120
  for (auto& o : inputs_) {
Q
qijun 已提交
121 122 123 124 125 126
    ret_val.reserve(ret_val.size() + o.second.size());
    ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
  }
  return ret_val;
}

Y
Yu Yang 已提交
127 128 129 130 131 132 133 134 135 136
std::vector<std::string> OperatorBase::OutputVars(bool has_intermediate) const {
  std::vector<std::string> ret_val;
  if (has_intermediate) {
    // push all outputs into ret_val
    for (auto& o : outputs_) {
      ret_val.reserve(ret_val.size() + o.second.size());
      ret_val.insert(ret_val.end(), o.second.begin(), o.second.end());
    }
    return ret_val;
  }
Y
Yu Yang 已提交
137
  auto& info = OpInfoMap::Instance().Get(Type());
Y
Yu Yang 已提交
138 139

  // get all OpProto::Var for outputs
Y
Yu Yang 已提交
140
  for (auto& o : info.Proto().outputs()) {
Y
Yu Yang 已提交
141 142 143 144 145 146 147 148 149
    // ignore all intermediate output
    if (o.intermediate()) continue;
    auto out = outputs_.find(o.name());
    if (out != outputs_.end()) {
      ret_val.reserve(ret_val.size() + out->second.size());
      ret_val.insert(ret_val.end(), out->second.begin(), out->second.end());
    }
  }
  return ret_val;
D
dongzhihong 已提交
150 151
}

152 153 154
void OperatorBase::CheckAllInputOutputSet() const {
  auto& info_map = OpInfoMap::Instance();
  auto* op_info = info_map.GetNullable(Type());
Y
Yu Yang 已提交
155
  if (op_info == nullptr || op_info->proto_ == nullptr) return;
156 157 158

  for (auto& in : op_info->Proto().inputs()) {
    PADDLE_ENFORCE(inputs_.find(in.name()) != inputs_.end(),
Y
Yu Yang 已提交
159
                   "Type %s's input %s is not set", Type(), in.name());
160 161 162 163
  }

  for (auto& out : op_info->Proto().outputs()) {
    PADDLE_ENFORCE(outputs_.find(out.name()) != outputs_.end(),
Y
Yu Yang 已提交
164
                   "Type %s's output %s is not set", Type(), out.name());
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
  }
}

void OperatorBase::GenerateTemporaryNames() {
  static std::atomic<size_t> gUniqId(0UL);
  for (auto& output : outputs_) {
    for (auto& output_name : output.second) {
      if (output_name == kTempVarName) {
        output_name += type_;
        output_name += "@";
        output_name += std::to_string(gUniqId.fetch_add(1));
      }
    }
  }
}

Q
QI JUN 已提交
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
static const Tensor* GetTensorFromVar(const Variable* var) {
  const Tensor* t = nullptr;
  if (var->IsType<LoDTensor>()) {
    t = &(var->Get<LoDTensor>());
  } else if (var->IsType<SelectedRows>()) {
    t = &(var->Get<SelectedRows>().value());
  } else {
    PADDLE_THROW("Variable type must be LoDTensor/SelectedRows.");
  }
  return t;
}

static Tensor* GetMutableTensorFromVar(Variable* var) {
  Tensor* t = nullptr;
  if (var->IsType<LoDTensor>()) {
    t = var->GetMutable<LoDTensor>();
  } else if (var->IsType<SelectedRows>()) {
    t = var->GetMutable<SelectedRows>()->mutable_value();
  } else {
    PADDLE_THROW("Variable type must be LoDTensor/SelectedRows.");
  }
  return t;
}

205
template <>
206
const Tensor* ExecutionContext::Input<Tensor>(const std::string& name) const {
207
  auto* var = InputVar(name);
208
  return var == nullptr ? nullptr : GetTensorFromVar(var);
209 210 211
}

template <>
212
const std::vector<const Tensor*> ExecutionContext::MultiInput<Tensor>(
213 214 215 216
    const std::string& name) const {
  auto names = op().Inputs(name);
  std::vector<const Tensor*> res;
  res.reserve(names.size());
217 218 219 220 221
  std::transform(names.begin(), names.end(), std::back_inserter(res),
                 [&](const std::string& sub_name) {
                   auto var = scope_.FindVar(sub_name);
                   return var == nullptr ? nullptr : GetTensorFromVar(var);
                 });
222 223 224 225
  return res;
}

template <>
226
Tensor* ExecutionContext::Output<Tensor>(const std::string& name) const {
227
  auto var = OutputVar(name);
Q
QI JUN 已提交
228
  return var == nullptr ? nullptr : GetMutableTensorFromVar(var);
229 230 231
}

template <>
232
std::vector<Tensor*> ExecutionContext::MultiOutput<Tensor>(
233 234 235 236
    const std::string& name) const {
  auto names = op().Outputs(name);
  std::vector<Tensor*> res;
  res.reserve(names.size());
237 238
  std::transform(names.begin(), names.end(), std::back_inserter(res),
                 [&](const std::string& sub_name) {
239 240
                   auto var = scope_.FindVar(sub_name);
                   return var == nullptr ? nullptr
Q
QI JUN 已提交
241
                                         : GetMutableTensorFromVar(var);
242
                 });
243 244 245
  return res;
}

Y
Yu Yang 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
bool OpSupportGPU(const std::string& op_type) {
  auto& all_kernels = OperatorWithKernel::AllOpKernels();
  auto it = all_kernels.find(op_type);
  if (it == all_kernels.end()) {
    // All control operator must support GPU
    return true;
  }
  for (auto& kern_pair : it->second) {
    if (platform::is_gpu_place(kern_pair.first.place_)) {
      return true;
    }
  }
  return false;
}

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
class RuntimeInferShapeContext : public InferShapeContext {
 public:
  RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope)
      : op_(op), scope_(scope) {}

  bool HasInput(const std::string& name) const override {
    auto& ins = Inputs(name);
    size_t length = ins.size();
    if (length == 0) {
      return false;
    }
    PADDLE_ENFORCE_EQ(length, 1UL, "Input %s should have more than one inputs",
                      name);
    auto ipt = ins[0];
    auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt);
    return var != nullptr;
  }

  bool HasOutput(const std::string& name) const override {
    auto& outs = Outputs(name);
    size_t length = outs.size();
    if (length == 0) {
      return false;
    }
    PADDLE_ENFORCE_EQ(length, 1UL, "Output %s should have more than one inputs",
                      name);
    auto ipt = outs[0];
    auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt);
    return var != nullptr;
  }

  bool HasInputs(const std::string& name) const override {
    auto inputs = op_.Inputs(name);
    if (inputs.empty()) {
      return false;
    }
    for (auto& input : inputs) {
      if (scope_.FindVar(input) == nullptr) {
        return false;
      }
    }
    return true;
  }

  bool HasOutputs(const std::string& name) const override {
    auto outputs = op_.Outputs(name);
    if (outputs.empty()) {
      return false;
    }
    for (auto& output : outputs) {
      if (scope_.FindVar(output) == nullptr) {
        return false;
      }
    }
    return true;
  }

  DDim GetInputDim(const std::string& name) const override {
    return GetDim(op_.Input(name));
  }

  void SetOutputDim(const std::string& name, const DDim& dim) override {
    SetDim(op_.Output(name), dim);
  }

  AttrReader Attrs() const override { return AttrReader(op_.Attrs()); }

  const std::vector<std::string>& Inputs(
      const std::string& name) const override {
    return op_.Inputs(name);
  }

  const std::vector<std::string>& Outputs(
      const std::string& name) const override {
    return op_.Outputs(name);
  }

Q
Qiao Longfei 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350 351
  void ShareLoD(const std::string& in, const std::string& out, size_t i = 0,
                size_t j = 0) const override {
    PADDLE_ENFORCE_LT(i, Inputs(in).size());
    PADDLE_ENFORCE_LT(j, Outputs(out).size());
    Variable* in_var = scope_.FindVar(Inputs(in)[i]);
    Variable* out_var = scope_.FindVar(Outputs(out)[j]);
    if (!in_var->IsType<LoDTensor>()) return;
    PADDLE_ENFORCE(out_var->IsType<LoDTensor>(),
                   "The %d-th output of Output(%s) must be LoDTensor.", j, out);
    auto in_tensor = in_var->Get<LoDTensor>();
    auto* out_tensor = out_var->GetMutable<LoDTensor>();
    out_tensor->set_lod(in_tensor.lod());
  }

352 353 354
  bool IsRuntime() const override { return true; }

 protected:
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
  DDim GetDim(const std::string& name) const override {
    Variable* var = scope_.FindVar(name);
    if (var->IsType<LoDTensor>()) {
      return var->Get<LoDTensor>().dims();
    } else if (var->IsType<SelectedRows>()) {
      return var->Get<SelectedRows>().GetCompleteDims();
    } else {
      PADDLE_THROW("Variable type must be LoDTensor/SelectedRows.");
    }
  }

  void SetDim(const std::string& name, const DDim& dim) override {
    Variable* var = scope_.FindVar(name);
    if (var->IsType<LoDTensor>()) {
      var->GetMutable<LoDTensor>()->Resize(dim);
    } else if (var->IsType<SelectedRows>()) {
      var->GetMutable<SelectedRows>()->set_height(dim[0]);
    } else {
      PADDLE_THROW("Variable type must be LoDTensor/SelectedRows.");
    }
  }

377
  proto::VarDesc::VarType GetVarType(const std::string& name) const override {
378 379 380 381 382
    auto* var = scope_.FindVar(name);
    return ToVarType(var->Type());
  }

 private:
383 384 385 386
  const OperatorBase& op_;
  const Scope& scope_;
};

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
const platform::DeviceContext* GetDeviceContext(
    framework::KernelTypePair& kernel_pair) {
  auto& actual_kernel_key = kernel_pair.first;
  auto& expected_kernel_key = kernel_pair.second;
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();

  if (platform::is_gpu_place(actual_kernel_key.place_) &&
      platform::is_cpu_place(expected_kernel_key.place_)) {
    return pool.Get(actual_kernel_key.place_);
  } else if (platform::is_cpu_place(actual_kernel_key.place_) &&
             platform::is_gpu_place(expected_kernel_key.place_)) {
    return pool.Get(expected_kernel_key.place_);
  } else {
    PADDLE_THROW(
        "Currently, model parallelism is only supported between CPU and CUDA");
  }
}

405
void OperatorWithKernel::Run(const Scope& scope,
D
dzhwinter 已提交
406
                             const platform::Place& place) const {
407 408
  RuntimeInferShapeContext infer_shape_ctx(*this, scope);
  this->InferShape(&infer_shape_ctx);
Y
Yu Yang 已提交
409 410
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
  auto dev_ctx = pool.Get(place);
411 412 413 414 415

  // check if op[type] has kernel registered.
  auto& all_op_kernels = AllOpKernels();
  auto kernels_iter = all_op_kernels.find(type_);
  if (kernels_iter == all_op_kernels.end()) {
Y
Yu Yang 已提交
416 417
    PADDLE_THROW(
        "There are no kernels which are registered in the %s operator.", type_);
418 419 420 421
  }

  // check if op[type] have kernel for kernel_key
  OpKernelMap& kernels = kernels_iter->second;
D
dzhwinter 已提交
422 423

  ExecutionContext ctx(*this, scope, *dev_ctx);
Q
Qiao Longfei 已提交
424 425 426
  auto actual_kernel_key = GetActualKernelType(ctx);
  auto expected_kernel_key = GetExpectedKernelType(actual_kernel_key);
  auto kernel_iter = kernels.find(expected_kernel_key);
427 428

  if (kernel_iter == kernels.end()) {
Q
Qiao Longfei 已提交
429 430
    PADDLE_THROW("The operator %s does not support %s", type_,
                 expected_kernel_key);
431 432
  }

433
  if (actual_kernel_key == expected_kernel_key) {
Q
QI JUN 已提交
434 435 436 437
    PADDLE_ENFORCE_EQ(actual_kernel_key.place_, expected_kernel_key.place_,
                      "Currently, model parallelism is only supported between "
                      "CPU and other devices. For example, multi-GPU model "
                      "parallelism will failed.");
438
  } else {
439
    auto kernel_pair = std::make_pair(actual_kernel_key, expected_kernel_key);
Q
QI JUN 已提交
440
    const DataTransformFn* trans_fun =
441
        DataTransformFnMap::Instance().GetNullable(kernel_pair);
Q
QI JUN 已提交
442 443 444 445 446 447 448 449 450 451 452 453 454 455
    if (trans_fun) {
      auto input_vars = this->InputVars();
      // TODO(qijun) filter the input vars that do not need to be transformed

      // filter vars that has been transformed
      std::vector<std::string> need_trans;
      for (auto var_name : input_vars) {
        auto var_name_trans =
            var_name + framework::KernelTypeToString(expected_kernel_key);
        if (!scope.FindVar(var_name_trans)) {
          const_cast<Scope&>(scope).Var(var_name_trans);
          need_trans.push_back(var_name);
        }
      }
456

Q
QI JUN 已提交
457
      if (!need_trans.empty()) {
458
        auto trans_dev_ctx = GetDeviceContext(kernel_pair);
Q
QI JUN 已提交
459 460 461 462 463

        // Wait for transform starting
        dev_ctx->Wait();

        for (auto var_name : need_trans) {
464
          (*trans_fun)(trans_dev_ctx, *(scope.FindVar(var_name)),
Q
QI JUN 已提交
465 466 467 468
                       scope.FindVar(var_name + framework::KernelTypeToString(
                                                    expected_kernel_key)));
        }
        // Wait for data transform finishing
469
        trans_dev_ctx->Wait();
Q
QI JUN 已提交
470
      }
471 472
    }
  }
Q
QI JUN 已提交
473 474

  kernel_iter->second->Compute(ctx);
475
}
Q
Qiao Longfei 已提交
476 477

OpKernelType OperatorWithKernel::GetActualKernelType(
Y
Yu Yang 已提交
478
    const ExecutionContext& ctx) const {
Q
QI JUN 已提交
479
  return OpKernelType(IndicateDataType(ctx), ctx.GetPlace());
Y
Yu Yang 已提交
480
}
Q
Qiao Longfei 已提交
481 482 483 484 485 486

OpKernelType OperatorWithKernel::GetExpectedKernelType(
    const OpKernelType& actual_kernel_type) const {
  return actual_kernel_type;
}

487
proto::DataType OperatorWithKernel::IndicateDataType(
Y
Yu Yang 已提交
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
    const ExecutionContext& ctx) const {
  auto& scope = ctx.scope();
  int data_type = -1;
  for (auto& input : this->inputs_) {
    for (auto& ipt_name : input.second) {
      auto* var = scope.FindVar(ipt_name);
      if (var != nullptr) {
        const Tensor* t = nullptr;
        if (var->IsType<Tensor>()) {
          t = &var->Get<Tensor>();
        } else if (var->IsType<LoDTensor>()) {
          t = &var->Get<LoDTensor>();
        } else if (var->IsType<SelectedRows>()) {
          t = &(var->Get<SelectedRows>().value());
        }
        if (t != nullptr) {
          int tmp = static_cast<int>(ToDataType(t->type()));
          PADDLE_ENFORCE(tmp == data_type || data_type == -1,
                         "DataType of Paddle Op %s must be the same.", Type());
          data_type = tmp;
        }
      }
    }
  }
  PADDLE_ENFORCE(data_type != -1, "DataType should be indicated by input");
513
  return static_cast<proto::DataType>(data_type);
Y
Yu Yang 已提交
514
}
515

Q
Qiao Longfei 已提交
516
}  // namespace framework
L
liaogang 已提交
517
}  // namespace paddle