executor.cpp 30.5 KB
Newer Older
W
wangliu 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

H
hjchen2 已提交
15
#include "framework/executor.h"
D
dolphin8 已提交
16
#include <algorithm>
17
#include <utility>
W
wangliu 已提交
18
#include <vector>
L
liuruilong 已提交
19
#include "common/enforce.h"
L
liuruilong 已提交
20
#include "common/log.h"
21
#include "framework/context.h"
L
liuruilong 已提交
22
#include "framework/framework.pb-c.h"
L
liuruilong 已提交
23 24
#include "framework/lod_tensor.h"
#include "framework/operator.h"
L
liuruilong 已提交
25
#include "framework/program/program-optimize/program_optimize.h"
L
liuruilong 已提交
26 27 28 29
#include "framework/program/program_desc.h"
#include "framework/program/var_desc.h"
#include "framework/scope.h"
#include "framework/tensor.h"
H
hjchen2 已提交
30
#include "memory/t_malloc.h"
H
hjchen2 已提交
31
#include "pass/memory_optimize.h"
L
update  
liuruilong 已提交
32 33 34
#ifdef PADDLE_MOBILE_CL
#include "framework/cl/cl_image.h"
#endif
W
wangliu 已提交
35 36

namespace paddle_mobile {
37
namespace framework {
38

W
wangliu 已提交
39 40
#pragma mark - executor

41 42 43 44 45
template <typename Device, typename T>
void Executor<Device, T>::SetThreadNum(int threads) {
  set_global_num_threads(threads);
}

46
template <typename Device, typename T>
xiebaiyuan's avatar
xiebaiyuan 已提交
47 48 49 50
Executor<Device, T>::Executor(const Program<Device> &program,
                              paddle_mobile::PaddleMobileConfigInternal config,
                              int batch_size, const bool use_optimize,
                              const bool lod_mode)
51
    : program_(program),
H
hjchen2 已提交
52 53
      batch_size_(batch_size),
      use_optimize_(use_optimize),
xiebaiyuan's avatar
xiebaiyuan 已提交
54 55
      lod_mode_(lod_mode),
      config_(config) {
56 57
  DLOG << "executor in lod mode: " << lod_mode_;

W
wangliu 已提交
58
  Variable *variable_ptr = program_.scope->Var("batch_size");
H
hjchen2 已提交
59
  variable_ptr->SetValue<int>(batch_size);
60 61

  program_desc_ =
Refine  
陈后江 已提交
62
      use_optimize_ ? program_.optimizeProgram : program_.originProgram;
63 64
  PADDLE_MOBILE_ENFORCE(program_desc_ != nullptr,
                        "program_desc_ should not be nullptr");
65 66
#if !defined(PADDLE_MOBILE_FPGA) && !defined(PADDLE_MOBILE_CL)
  pass::MemoryOptPass()(program_desc_.get(), program_.scope.get());
67
#endif
68 69 70
  // resize feed and fetch list
  // should init feed and fetch variables before infer shape
  InitFeedFetchList();
71

72
  const auto &blocks = program_desc_->Blocks();
73 74 75 76 77 78 79 80
  std::shared_ptr<BlockDesc> block_desc = blocks[0];
  std::vector<std::shared_ptr<OpDesc>> ops = block_desc->Ops();
  for (int j = 0; j < ops.size(); ++j) {
    std::shared_ptr<OpDesc> op_desc = ops[j];
    DLOG << "create op: " << op_desc->Type();

    auto op_handler = OpRegistry<Device>::CreateOp(
        op_desc->Type(), op_desc->GetInputs(), op_desc->GetOutputs(),
81
        op_desc->GetAttrMap(), program_.scope.get());
82 83 84 85
    // infer shape to reshape inputs and outputs before predict,
    // but for lod mode, it still need to infer shape in runtime
    if (!lod_mode) {
      op_handler->InferShape();
W
wangliu 已提交
86
    }
87
    ops_of_block0_.push_back(op_handler);
W
wangliu 已提交
88
  }
W
wangliu 已提交
89
  if (program_.combined) {
L
liuruilong 已提交
90 91 92 93
    InitCombineMemory();
  } else {
    InitMemory();
  }
94 95

  int count = 0;
96 97 98
  for (auto &op_handler : ops_of_block0_) {
    DLOG << "Initialize op[" << count++ << "]: " << op_handler->Type();
    op_handler->Init();
L
liuruilong 已提交
99
  }
W
wangliu 已提交
100 101
}

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
template <typename Device, typename T>
void Executor<Device, T>::InitFeedFetchList() {
  std::unordered_map<std::string, int> feed_indices, fetch_indices;
  for (const auto &block : program_desc_->Blocks()) {
    for (const auto &op_desc : block->Ops()) {
      if (op_desc->Type() == "feed") {
        std::string name = op_desc->Output("Out")[0];
        feed_indices[name] = op_desc->GetAttr("col").Get<int>();
      } else if (op_desc->Type() == "fetch") {
        std::string name = op_desc->Input("X")[0];
        fetch_indices[name] = op_desc->GetAttr("col").Get<int>();
      }
    }
  }
  feed_indices_.swap(feed_indices);
  fetch_indices_.swap(fetch_indices);

  auto *feed_var = program_.scope->Var("feed");
  auto *feed_list = feed_var->template GetMutable<framework::LoDTensorArray>();
  feed_list->resize(feed_indices_.size());

  auto *fetch_var = program_.scope->Var("fetch");
  auto *fetch_list =
      fetch_var->template GetMutable<framework::LoDTensorArray>();
  fetch_list->resize(fetch_indices_.size());
}

129
template <typename T>
130
static void LoadMemInternal(void **data, LoDTensor *tensor,
131
                            bool quant_uint8 = false) {
Refine  
陈后江 已提交
132
  char **data_buf = reinterpret_cast<char **>(data);
133
  int64_t size = tensor->numel();
134
  T *tensor_data = tensor->mutable_data<T>();
135 136
  if (quant_uint8) {
    // should be moved into operator init function
137 138
    float min_value;
    float max_value;
139 140 141
    memory::Copy(&min_value, *data_buf, sizeof(float));
    memory::Copy(&max_value, *data_buf + sizeof(float), sizeof(float));
    *data_buf += 2 * sizeof(float);
142
    const float factor = (max_value - min_value) / 255.0;
143
    const uint8_t *uint8_data = reinterpret_cast<uint8_t *>(*data_buf);
144 145
    for (int k = 0; k < size; ++k) {
      tensor_data[k] = uint8_data[k] * factor + min_value;
W
wangliu 已提交
146
    }
147
    *data_buf += size * sizeof(uint8_t);
148
  } else {
149 150
    memory::Copy(tensor_data, *data_buf, size * sizeof(T));
    *data_buf += size * sizeof(T);
L
liuruilong 已提交
151
  }
152
}
W
wangliu 已提交
153

154 155 156 157
template <typename Device, typename T>
void Executor<Device, T>::LoadMemory(void **data,
                                     const std::shared_ptr<VarDesc> var_desc,
                                     LoDTensor *tensor) {
158
  char **data_buf = reinterpret_cast<char **>(data);
159
  // version
160
  uint32_t version = *(reinterpret_cast<uint32_t *>(*data_buf));
Refine  
陈后江 已提交
161
  *data_buf += sizeof(uint32_t);
162
  // lod information
H
hjchen2 已提交
163 164
  // uint64_t lod_level = *(reinterpret_cast<uint64_t *>(*data_buf));
  uint64_t lod_level = 0;
Z
zhangyang 已提交
165
  memory::Copy(&lod_level, *data_buf, sizeof(uint64_t));
Refine  
陈后江 已提交
166
  *data_buf += sizeof(uint64_t);
167 168 169 170

  auto *lod = tensor->mutable_lod();
  lod->resize(lod_level);
  for (uint64_t i = 0; i < lod_level; ++i) {
171
    uint64_t size = *(reinterpret_cast<uint64_t *>(*data_buf));
Refine  
陈后江 已提交
172
    *data_buf += sizeof(uint64_t);
173
    std::vector<size_t> tmp_dim(size / sizeof(size_t));
Z
zhangyang 已提交
174
    memory::Copy(tmp_dim.data(), *data_buf, size);
175
    (*lod)[i] = std::move(tmp_dim);
Refine  
陈后江 已提交
176
    *data_buf += size;
W
wangliu 已提交
177
  }
178
  // tensor version
179
  uint32_t tensor_version = *(reinterpret_cast<uint32_t *>(*data_buf));
Refine  
陈后江 已提交
180
  *data_buf += sizeof(uint32_t);
181
  // tensor desc size
182
  int32_t tensor_desc_size = *(reinterpret_cast<int32_t *>(*data_buf));
Refine  
陈后江 已提交
183
  *data_buf += sizeof(int32_t);
184
  // skip tensor desc
Refine  
陈后江 已提交
185
  *data_buf += tensor_desc_size;
186

187 188
  const TensorDesc &tensor_desc = var_desc->Tensor_desc();
  tensor->Resize(make_ddim(tensor_desc.Dims()));
189 190
  // parse tensor from stream
  switch (tensor_desc.DataType()) {
191
    case VARTYPE_TYPE_FP32:
192 193
      LoadMemInternal<float>(reinterpret_cast<void **>(data_buf), tensor,
                             program_.quantification);
W
wangliu 已提交
194
      break;
195
    case VARTYPE_TYPE_INT8:
196
      LoadMemInternal<int8_t>(reinterpret_cast<void **>(data_buf), tensor);
W
wangliu 已提交
197
      break;
198
    case VARTYPE_TYPE_INT32:
199
      LoadMemInternal<int>(reinterpret_cast<void **>(data_buf), tensor);
W
wangliu 已提交
200 201
      break;
    default:
202
      LOG(kLOG_ERROR) << "data type is not supported";
L
liuruilong 已提交
203
  }
W
wangliu 已提交
204 205
}

206 207 208
template <typename Device, typename T>
void Executor<Device, T>::InitMemory() {
  for (const auto &block : program_desc_->Blocks()) {
W
wangliu 已提交
209 210 211 212
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());
      if (var_desc->Persistable()) {
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
H
update  
hjchen2 已提交
213
          var->template GetMutable<framework::LoDTensorArray>();
W
wangliu 已提交
214 215
          continue;
        }
H
hjchen2 已提交
216
        DLOG << "init persistable var: " << var_desc->Name();
Refine  
陈后江 已提交
217
        char *origin_data =
Refine  
陈后江 已提交
218
            ReadFileToBuff(program_.model_path + "/" + var_desc->Name());
Refine  
陈后江 已提交
219
        char *data = origin_data;
H
update  
hjchen2 已提交
220
        auto tensor = var->template GetMutable<LoDTensor>();
221 222
        LoadMemory(reinterpret_cast<void **>(&data), var_desc, tensor);
        delete[] origin_data;
W
wangliu 已提交
223
      } else {
224
        DLOG << "init no persistable var: " << var_desc->Name();
H
update  
hjchen2 已提交
225
        varInputMemory(var_desc, var);
W
wangliu 已提交
226 227 228 229 230
      }
    }
  }
}

231 232
template <typename Device, typename T>
void Executor<Device, T>::InitCombineMemory() {
Refine  
陈后江 已提交
233
  char *origin_data = nullptr;
Refine  
陈后江 已提交
234
  bool self_alloc = false;
235
  if (program_.combined_params_buf && program_.combined_params_len) {
236 237
    origin_data = reinterpret_cast<char *>(
        const_cast<uint8_t *>(program_.combined_params_buf));
238
  } else {
Refine  
陈后江 已提交
239
    self_alloc = true;
Refine  
陈后江 已提交
240
    origin_data = ReadFileToBuff(program_.para_path);
241
  }
Refine  
陈后江 已提交
242 243
  PADDLE_MOBILE_ENFORCE(origin_data != nullptr, "data == nullptr");
  char *data = origin_data;
244
  for (const auto &block : program_desc_->Blocks()) {
L
liuruilong 已提交
245 246 247 248
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());
      if (var_desc->Persistable()) {
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
H
update  
hjchen2 已提交
249
          var->template GetMutable<framework::LoDTensorArray>();
L
liuruilong 已提交
250 251
          continue;
        }
L
liuruilong 已提交
252 253

        DLOG << " init combine memory persistable: " << var_desc->Name();
H
update  
hjchen2 已提交
254
        auto tensor = var->template GetMutable<LoDTensor>();
255
        LoadMemory(reinterpret_cast<void **>(&data), var_desc, tensor);
L
liuruilong 已提交
256
      } else {
H
update  
hjchen2 已提交
257 258
        DLOG << " init combine memory no persistable: " << var_desc->Name();
        varInputMemory(var_desc, var);
L
liuruilong 已提交
259 260 261
      }
    }
  }
Refine  
陈后江 已提交
262
  if (self_alloc) {
263
    delete[] origin_data;
Refine  
陈后江 已提交
264 265
  }
  LOG(kLOG_INFO) << "init combine memory finish";
L
liuruilong 已提交
266
}
267

L
liuruilong 已提交
268
template <typename Device, typename T>
L
liuruilong 已提交
269
void Executor<Device, T>::InitNoPersistableMemory(const Tensor &input_tensor) {
L
liuruilong 已提交
270 271 272 273 274 275
  for (const auto &block : program_desc_->Blocks()) {
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());
      auto tensor = var->template GetMutable<LoDTensor>();
      if (var_desc->Persistable()) {
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
H
update  
hjchen2 已提交
276
          var->template GetMutable<framework::LoDTensorArray>();
L
liuruilong 已提交
277 278 279 280 281
          continue;
        }
      } else {
        if (var_desc->Type() == VARTYPE_TYPE_LOD_TENSOR) {
          DDim tensor_dim = tensor->dims();
xiebaiyuan's avatar
xiebaiyuan 已提交
282 283 284 285
          DDim new_dim =
              make_ddim({tensor_dim[0], tensor_dim[1], input_tensor.dims()[2],
                         input_tensor.dims()[3]});
          tensor->Resize(new_dim);
L
liuruilong 已提交
286
          tensor->template mutable_data<T>();
H
update  
hjchen2 已提交
287 288 289
        } else {
          PADDLE_MOBILE_THROW_EXCEPTION("Unsupported var type `%d`",
                                        var_desc->Type());
L
liuruilong 已提交
290 291 292 293 294 295 296 297 298 299
        }
      }
    }
  }

  std::shared_ptr<LoDTensor> output = GetOutput("fetch");
  output->Resize(input_tensor.dims());
  output->mutable_data<T>();
}

300 301
template <typename Device, typename T>
bool Executor<Device, T>::varInputMemory(
H
update  
hjchen2 已提交
302
    const std::shared_ptr<VarDesc> &var_desc, Variable *var) const {
303
#ifdef PADDLE_MOBILE_FPGA
H
hjchen2 已提交
304
  framework::LoDTensor *tensor = var->template GetMutable<LoDTensor>();
305
  tensor->init(type_id<float>().hash_code());
306 307
  return true;
#endif
H
update  
hjchen2 已提交
308 309 310 311 312 313 314 315 316 317 318 319 320

  auto type = var_desc->Type();
  if (type == VARTYPE_TYPE_LOD_TENSOR) {
    auto data_type = var_desc->Tensor_desc().DataType();
    framework::LoDTensor *tensor = var->template GetMutable<LoDTensor>();
  } else if (type == VARTYPE_TYPE_STEP_SCOPES) {
    std::vector<framework::Scope *> *step_scopes =
        var->template GetMutable<std::vector<framework::Scope *>>();
  } else if (type == VARTYPE_TYPE_STEP_LOD_TENSOR_ARRAY) {
    framework::LoDTensorArray *tensor_array =
        var->template GetMutable<framework::LoDTensorArray>();
  } else {
    PADDLE_MOBILE_THROW_EXCEPTION("got unhandled var type `%d`", type);
xiebaiyuan's avatar
xiebaiyuan 已提交
321
  }
H
update  
hjchen2 已提交
322
  return true;
xiebaiyuan's avatar
xiebaiyuan 已提交
323
}
L
liuruilong 已提交
324

325 326 327 328 329
template <typename Device, typename T>
PMStatus Executor<Device, T>::Predict(
    const std::vector<std::pair<std::string, Tensor>> &inputs) {
  for (const auto &input : inputs) {
    SetInput(input.second, input.first);
D
dolphin8 已提交
330
  }
331 332 333 334 335 336 337 338
  return this->Predict();
}

template <typename Device, typename T>
PMStatus Executor<Device, T>::Predict(
    const std::vector<std::pair<std::string, LoDTensor>> &inputs) {
  for (const auto &input : inputs) {
    SetInput(input.second, input.first);
D
dolphin8 已提交
339
  }
340
  return this->Predict();
W
wangliu 已提交
341
}
xiebaiyuan's avatar
xiebaiyuan 已提交
342

343 344 345
template <typename Device, typename T>
std::vector<T> Executor<Device, T>::Predict(const std::vector<T> &input,
                                            const std::vector<int64_t> &dims) {
346 347 348 349 350 351 352
  PADDLE_MOBILE_ENFORCE(feed_indices_.size() != 0,
                        "We don't know which tensor should be assign, since no "
                        "feed op found in this model");
  PADDLE_MOBILE_ENFORCE(fetch_indices_.size() != 0,
                        "We don't know which tensor should be fetch out, since "
                        "no fetch op found in this model");
  std::string input_name = feed_indices_.begin()->first;
353
  Tensor feed_tensor(input, make_ddim(dims));
354
  SetInput(feed_tensor, input_name);
355 356
  std::vector<T> output;
  if (this->Predict() == PMSuccess) {
357 358
    std::string output_name = fetch_indices_.begin()->first;
    const auto output_tensor = GetOutput(output_name);
359 360 361 362 363 364
    output.resize(output_tensor->numel());
    memcpy(output.data(), output_tensor->template data<T>(),
           output.size() * sizeof(T));
  }
  return output;
}
xiebaiyuan's avatar
xiebaiyuan 已提交
365

366 367 368
template <typename Device, typename T>
void Executor<Device, T>::SetInput(const Tensor &input,
                                   const std::string &var_name) {
H
hjchen2 已提交
369
  int index = 0;
370
  if (feed_indices_.find(var_name) != feed_indices_.end()) {
H
hjchen2 已提交
371
    index = feed_indices_.find(var_name)->second;
372
  }
H
hjchen2 已提交
373 374 375 376 377 378
  auto *feed_var = program_.scope->Var("feed");
  framework::LoDTensor &target =
      feed_var->template GetMutable<framework::LoDTensorArray>()->at(index);

  target.Resize(input.dims());
  target.ShareDataWith(input);
379
}
xiebaiyuan's avatar
xiebaiyuan 已提交
380

381 382 383
template <typename Device, typename T>
void Executor<Device, T>::SetInput(const LoDTensor &input,
                                   const std::string &var_name) {
H
hjchen2 已提交
384
  int index = 0;
385
  if (feed_indices_.find(var_name) != feed_indices_.end()) {
H
hjchen2 已提交
386
    index = feed_indices_.find(var_name)->second;
387
  }
H
hjchen2 已提交
388 389 390 391 392 393 394
  auto *feed_var = program_.scope->Var("feed");
  framework::LoDTensor &target =
      feed_var->template GetMutable<framework::LoDTensorArray>()->at(index);

  target.Resize(input.dims());
  target.ShareDataWith(input);
  target.set_lod(input.lod());
395 396 397 398 399
}

template <typename Device, typename T>
std::shared_ptr<LoDTensor> Executor<Device, T>::GetOutput(
    const std::string &var_name) {
400 401 402 403 404 405 406 407 408
  const auto &iter = fetch_indices_.find(var_name);
  if (var_name == "fetch" || iter != fetch_indices_.end()) {
    int index = 0;
    if (iter != fetch_indices_.end()) {
      index = iter->second;
    }
    auto *fetch_var = program_.scope->Var("fetch");
    framework::LoDTensor &target =
        fetch_var->template GetMutable<framework::LoDTensorArray>()->at(index);
H
hjchen2 已提交
409

410 411 412 413 414 415 416
    return std::make_shared<LoDTensor>(target);
  } else {
    auto *fetch_var = program_.scope->Var(var_name);
    framework::LoDTensor *target =
        fetch_var->template GetMutable<framework::LoDTensor>();
    return std::make_shared<LoDTensor>(*target);
  }
417
}
xiebaiyuan's avatar
xiebaiyuan 已提交
418

419 420
template <typename Device, typename T>
PMStatus Executor<Device, T>::Predict() {
421 422 423
#if _OPENMP
  omp_set_num_threads(get_global_num_threads());
#endif
xiebaiyuan's avatar
xiebaiyuan 已提交
424
#ifdef PADDLE_MOBILE_PROFILE
425
  std::vector<ProfInfo> profile(ops_of_block0_.size());
426 427
  struct timespec ts;
  int op_index = 0;
xiebaiyuan's avatar
xiebaiyuan 已提交
428
#endif
429
  for (auto &op_handler : ops_of_block0_) {
xiebaiyuan's avatar
xiebaiyuan 已提交
430
#ifdef PADDLE_MOBILE_PROFILE
431 432
    clock_gettime(CLOCK_MONOTONIC, &ts);
    profile[op_index].runBegin = (uint64_t)ts.tv_sec * 1e9 + ts.tv_nsec;
xiebaiyuan's avatar
xiebaiyuan 已提交
433
#endif
H
hjchen2 已提交
434
    DLOG << "run op: " << op_handler->Type();
435 436 437 438
    if (lod_mode_) {
      op_handler->InferShape();
    }
    op_handler->Run();
xiebaiyuan's avatar
xiebaiyuan 已提交
439
#ifdef PADDLE_MOBILE_PROFILE
440 441 442
    clock_gettime(CLOCK_MONOTONIC, &ts);
    profile[op_index].runEnd = (uint64_t)ts.tv_sec * 1e9 + ts.tv_nsec;
    ++op_index;
xiebaiyuan's avatar
xiebaiyuan 已提交
443 444 445 446 447 448 449
#endif
  }
#ifdef PADDLE_MOBILE_PROFILE
  std::unordered_map<std::string, uint64_t> _tp;
  for (int i = 0; i < profile.size(); i++) {
    const auto &pInfo = profile[i];
    uint64_t timeCost = pInfo.runEnd - pInfo.runBegin;
450 451 452
    if (ops_of_block0_[i]->Type() == "conv2d" ||
        ops_of_block0_[i]->Type() == "depthwise_conv2d") {
      auto inputs = ops_of_block0_[i]->Inputs();
453 454
      auto *filter =
          GetVarValue<LoDTensor>("Filter", inputs, *(program_.scope));
455
      int kernel_size = filter->dims()[2];
456 457
      _tp[ops_of_block0_[i]->Type() + "_" + std::to_string(kernel_size)] +=
          timeCost;
458
    } else {
459
      _tp[ops_of_block0_[i]->Type()] += timeCost;
460
    }
xiebaiyuan's avatar
xiebaiyuan 已提交
461
  }
H
hjchen2 已提交
462
  printf("====================[ profile ]======================\n");
463
  typedef std::pair<std::string, uint64_t> prof_t;
xiebaiyuan's avatar
xiebaiyuan 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
  std::vector<prof_t> _tv(_tp.begin(), _tp.end());
  uint64_t _ptotal = 0;
  for (auto const &p : _tv) {
    _ptotal += p.second;
  }
  auto compf = [](const prof_t &a, const prof_t &b) {
    return a.second > b.second;
  };
  std::sort(_tv.begin(), _tv.end(), compf);
  _tv.push_back(std::make_pair("total", _ptotal));
  for (auto const &p : _tv) {
    printf("%-16s\t%-10.0f\t%-2.4f\n", p.first.c_str(),
           static_cast<float>(p.second),
           static_cast<float>(p.second) / _ptotal * 100.0);
  }
H
hjchen2 已提交
479
  printf("====================[---------]======================\n");
xiebaiyuan's avatar
xiebaiyuan 已提交
480
#endif
481
  return PMSuccess;
xiebaiyuan's avatar
xiebaiyuan 已提交
482 483
}

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
template <typename Device, typename T>
void Executor<Device, T>::FeedTensorData(const vector<framework::Tensor> &v) {
  auto input_size = v.size();
  auto *feed_var = program_.scope->Var("feed");

  PADDLE_MOBILE_ENFORCE(input_size == feed_indices_.size(),
                        "input data number not correct");
  for (int i = 0; i < input_size; i++) {
    framework::LoDTensor &target =
        feed_var->template GetMutable<framework::LoDTensorArray>()->at(i);
    target.ShareDataWith(v[input_size - i - 1]);
  }
}

template <typename Device, typename T>
void Executor<Device, T>::GetTensorResults(
    std::vector<framework::Tensor *> *v) {
  auto *fetch_var = program_.scope->Var("fetch");
  auto output_size = fetch_indices_.size();
  for (int i = 0; i < output_size; i++) {
    framework::LoDTensor &target =
        fetch_var->template GetMutable<framework::LoDTensorArray>()->at(i);
    v->push_back(&target);
  }
}

510
#ifdef PADDLE_MOBILE_FPGA
511 512 513 514
template <typename Device, typename T>
void Executor<Device, T>::InjectVariable(const Tensor &t,
                                         std::string var_name) {
  Variable *g_feed_value = program_.scope->Var(var_name);
515
  Tensor *feed_tensor = g_feed_value->template GetMutable<LoDTensor>();
516 517
  feed_tensor->Resize(t.dims());
  feed_tensor->ShareDataWith(t);
518
}
519

520 521
template <typename Device, typename T>
void Executor<Device, T>::FeedData(const Tensor &t) {
Z
zhangyang0701 已提交
522
  InjectVariable(t, "feed0");
523
}
524

525
template <typename Device, typename T>
526
void Executor<Device, T>::FeedData(const std::vector<void *> &v) {
527
  auto input_size = v.size();
Z
zhangyang0701 已提交
528 529
  int index = 0;
  auto vars = program_.scope->VarContain("feed", &index);
530 531 532
  PADDLE_MOBILE_ENFORCE(input_size == vars.size(),
                        "input data number not correct");
  for (int i = 0; i < input_size; i++) {
Z
zhangyang0701 已提交
533
    auto var = program_.scope->Var("feed", i + index);
534 535 536 537 538 539 540 541 542
    auto feed_tensor = var->template GetMutable<LoDTensor>();
    feed_tensor->external_data = v[i];
  }
}

template <typename Device, typename T>
void Executor<Device, T>::GetResults(std::vector<void *> *v) {
  auto output_size = v->size();
  PADDLE_MOBILE_ENFORCE(output_size > 0, "Empty output");
Z
zhangyang0701 已提交
543 544
  int index = 0;
  auto vars = program_.scope->VarContain("fetch", &index);
545 546
  PADDLE_MOBILE_ENFORCE(output_size == vars.size(),
                        "output data number not correct");
547

548
  for (int i = 0; i < output_size; i++) {
Z
zhangyang0701 已提交
549
    auto var = program_.scope->Var("fetch", i + index);
550 551
    auto fetch_tensor = var->template GetMutable<LoDTensor>();
    (*v)[i] = fetch_tensor->template data<float>();
552
  }
553
}
554

555
template <typename Device, typename T>
556 557 558 559
framework::Tensor *Executor<Device, T>::GetTensorByName(
    const std::string &name) {
  auto var = program_.scope->Var(name);
  return var->template GetMutable<LoDTensor>();
H
hjchen2 已提交
560
}
561

562 563
template <typename Device, typename T>
std::shared_ptr<Tensor> Executor<Device, T>::FetchResult(int id) {
564
  auto &ops = ops_of_block0_;
565

Z
zhangyang 已提交
566 567 568 569 570
  PADDLE_MOBILE_ENFORCE(id < (int)ops.size(), "Index out of range");
  auto op = id < 0 ? ops[ops.size() - 1] : ops[id];
  auto output_map = op->Outputs();
  std::vector<std::string> out_keys = op->GetOutKeys();
  PADDLE_MOBILE_ENFORCE(!out_keys.empty(), "this op contains no output");
571 572 573
  auto *output_tensor =
      GetVarValue<LoDTensor>(out_keys[0], output_map, *(program_.scope));
  return std::make_shared<Tensor>(Tensor(*output_tensor));
574
}
575

576 577
template <typename Device, typename T>
void Executor<Device, T>::Predict_From_To(int start, int end) {
578
  auto &ops = ops_of_block0_;
579
  end = end < 0 ? static_cast<int>(ops.size()) : end;
580 581 582 583 584 585 586 587 588 589 590 591
  PADDLE_MOBILE_ENFORCE(start >= 0 && start < end && end <= ops.size(),
                        "start or end parameter is wrong");

#ifdef PADDLE_MOBILE_PROFILE
  std::vector<ProfInfo> profile(ops.size());
#endif
  for (int i = start; i < end; i++) {
#ifdef PADDLE_MOBILE_PROFILE
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    profile[i].runBegin = (uint64_t)ts.tv_sec * 1e9 + ts.tv_nsec;
#endif
Z
zhangyang 已提交
592
    DLOG << "Running op: " << i << "  " << ops[i]->Type();
593 594 595 596 597 598 599
    ops[i]->Run();

#ifdef PADDLE_MOBILE_PROFILE
    clock_gettime(CLOCK_MONOTONIC, &ts);
    profile[i].runEnd = (uint64_t)ts.tv_sec * 1e9 + ts.tv_nsec;
#endif
  }
600
}
601

602 603
template <typename Device, typename T>
void Executor<Device, T>::Predict_From(int start) {
604
  Predict_From_To(start);
605
}
606

607 608
template <typename Device, typename T>
void Executor<Device, T>::Predict_To(int end) {
609
  Predict_From_To(0, end);
610
}
611 612
#endif

Y
yangfei 已提交
613
#ifdef PADDLE_MOBILE_CL
xiebaiyuan's avatar
xiebaiyuan 已提交
614 615
template <>
void Executor<GPU_CL, float>::InitNoPersistableMemory(
616
    const Tensor &input_tensor) {
xiebaiyuan's avatar
xiebaiyuan 已提交
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
  DLOG << "CL InitNoPersistableMemory ";
  for (const auto &block : program_desc_->Blocks()) {
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());

      auto cl_image = var->template GetMutable<CLImage>();

      if (var_desc->Persistable()) {
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
          continue;
        }
      } else {
        if (var_desc->Type() == VARTYPE_TYPE_LOD_TENSOR) {
          cl_context context = program_.scope->GetCLScpoe()->Context();
          cl_command_queue command_queue =
              program_.scope->GetCLScpoe()->CommandQueue();

          DDim tensor_dim = cl_image->dims();
          DDim new_dim =
              make_ddim({tensor_dim[0], tensor_dim[1], input_tensor.dims()[2],
                         input_tensor.dims()[3]});
          cl_image->Resize(new_dim);
          cl_image->InitEmptyImage(context, command_queue, new_dim);
        }
      }
    }
  }
  std::shared_ptr<LoDTensor> output = GetOutput("fetch");
  output->Resize(input_tensor.dims());
  output->mutable_data<float>();
}
H
hjchen2 已提交
648

xiebaiyuan's avatar
xiebaiyuan 已提交
649 650 651
template <>
void Executor<GPU_CL, float>::SetInput(const Tensor &input,
                                       const std::string &var_name) {
H
hjchen2 已提交
652 653 654 655 656 657 658
  int index = 0;
  if (feed_indices_.find(var_name) != feed_indices_.end()) {
    index = feed_indices_.find(var_name)->second;
  }
  auto *feed_var = program_.scope->Var("feed");
  framework::LoDTensor *target_tensor =
      &(feed_var->template GetMutable<framework::LoDTensorArray>()->at(index));
xiebaiyuan's avatar
xiebaiyuan 已提交
659 660 661 662 663

  DLOG << "config_.load_when_predict   " << config_.load_when_predict;
  DLOG << "target_tensor->IsInitialized() " << target_tensor->IsInitialized();
  DLOG << "target_tensor->dims()   " << target_tensor->dims();
  DLOG << "input.dims()   " << input.dims();
664
  DLOG << "input_dim_last_   " << input_dim_last_;
xiebaiyuan's avatar
xiebaiyuan 已提交
665
  if (config_.load_when_predict) {
xiebaiyuan's avatar
xiebaiyuan 已提交
666
    if (input_dim_last_ != input.dims()) {
667 668 669
      DLOG << "SetInput ---- > resize1";
      target_tensor->Resize(input.dims());
      target_tensor->mutable_data<float>();
xiebaiyuan's avatar
xiebaiyuan 已提交
670 671 672 673 674 675 676 677
      InitNoPersistableMemory(*target_tensor);
    }
  } else {
    DLOG << "SetInput ---- > resize2";
    target_tensor->Resize(input.dims());
    DLOG << "SetInput ---- > ShareDataWith";
  }
  target_tensor->ShareDataWith(input);
678 679
  auto &dim = input.dims();
  input_dim_last_ = static_cast<DDim>(dim);
xiebaiyuan's avatar
xiebaiyuan 已提交
680 681
}

682 683 684
template <typename Device, typename T>
void Executor<Device, T>::LoadMemory(const VarDesc var_desc, float *tensorInput,
                                     char **data) {}
L
liuruilong 已提交
685

Y
yangfei 已提交
686
template <>
H
hjchen2 已提交
687 688
void Executor<GPU_CL, float>::LoadMemory(const VarDesc var_desc,
                                         float *tensorInput, char **data) {
689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
  // 1. version
  uint32_t version = *reinterpret_cast<uint32_t *>(*data);

  (*data) += sizeof(uint32_t);

  // 2 Lod information
  uint64_t *lod_level_ptr = new uint64_t();
  memcpy(lod_level_ptr, (*data), sizeof(uint64_t));
  uint64_t lod_level = *lod_level_ptr;
  delete lod_level_ptr;
  (*data) += sizeof(uint64_t);

  for (uint64_t i = 0; i < lod_level; ++i) {
    uint64_t size = *reinterpret_cast<uint64_t *>(*data);
    (*data) += sizeof(uint64_t);
    std::vector<size_t> tmp(size / sizeof(size_t));

    for (int k = 0; k < tmp.size(); ++k) {
      tmp[k] = *reinterpret_cast<size_t *>(*data);
      (*data) += sizeof(size_t);
    }
  }

  // 3. tensor version
  uint32_t tensor_version = *reinterpret_cast<uint32_t *>(*data);
  (*data) += sizeof(uint32_t);

  // 4. tensor desc
  int32_t size = *reinterpret_cast<int32_t *>(*data);
  (*data) += sizeof(int32_t);

  std::unique_ptr<char[]> buf(new char[size]);
  for (int m = 0; m < size; ++m) {
    buf.get()[m] = (*data)[m];
  }
  (*data) += (sizeof(char) * size);

726
  const TensorDesc &desc = var_desc.Tensor_desc();
727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
  int memory_size = 1;
  for (auto l : desc.Dims()) {
    memory_size *= l;
  }

  void *memory = nullptr;
  int type_size = 4;
  memory = tensorInput;
  if (program_.quantification) {
    float min_value;
    float max_value;

    memcpy(&min_value, *data, sizeof(float));
    memcpy(&max_value, *data + sizeof(float), sizeof(float));
    *data += 2 * sizeof(float);
    const float factor = (max_value - min_value) / 255.0;
    uint8_t *uint8_data = reinterpret_cast<uint8_t *>(*data);
    for (int k = 0; k < memory_size; ++k) {
      static_cast<float *>(memory)[k] = uint8_data[k] * factor + min_value;
    }
    *data += (memory_size * sizeof(uint8_t));
  } else {
    for (int n = 0; n < memory_size; n++) {
      float value;
      memcpy(&value, *data + n * type_size, type_size);
      if (value < 1e-30 && value > -1e-30) {
        static_cast<float *>(memory)[n] = 0.0;
      } else {
        static_cast<float *>(memory)[n] = value;
      }
    }
    (*data) += (sizeof(char) * memory_size * type_size);
  }
}
761

Y
yangfei 已提交
762
template <>
763 764
void Executor<GPU_CL, float>::InitMemory() {
  for (const auto &block : program_desc_->Blocks()) {
Y
yangfei 已提交
765 766 767
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());
      if (var_desc->Persistable()) {
L
liuruilong 已提交
768
        CLImage *cl_image = nullptr;
Y
yangfei 已提交
769
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
H
hjchen2 已提交
770
          var->template GetMutable<framework::LoDTensorArray>();
Y
yangfei 已提交
771
          continue;
L
liuruilong 已提交
772
        } else {
773
          cl_image = var->template GetMutable<CLImage>();
Y
yangfei 已提交
774
        }
L
liuruilong 已提交
775

Y
yangfei 已提交
776
        char *origin_data =
L
liuruilong 已提交
777
            ReadFileToBuff(program_.model_path + "/" + var_desc->Name());
778
        char *data = origin_data;
Y
yangfei 已提交
779
        cl_context context = program_.scope->GetCLScpoe()->Context();
780
        const TensorDesc &desc = var_desc->Tensor_desc();
781 782 783 784 785
        int numel = 1;
        for (auto l : desc.Dims()) {
          numel *= l;
        }
        DLOG << var_desc->Name();
Y
yangfei 已提交
786
        float *tensorInput = static_cast<float *>(
787 788
            paddle_mobile::memory::Alloc(sizeof(float) * numel));
        LoadMemory(*var_desc, tensorInput, &data);
Y
yangfei 已提交
789

790
        DDim ddim = make_ddim(desc.Dims());
Y
yangfei 已提交
791

L
liuruilong 已提交
792 793
        // has not init
        cl_image->SetTensorData(tensorInput, ddim);
Y
yangfei 已提交
794

795
        delete origin_data;
Y
yangfei 已提交
796
        paddle_mobile::memory::Free(tensorInput);
797
      } else {
798 799
        if (var_desc->Type() == VARTYPE_TYPE_LOD_TENSOR) {
          auto cl_image = var->template GetMutable<CLImage>();
800
          cl_context context = program_.scope->GetCLScpoe()->Context();
L
liuruilong 已提交
801 802
          cl_command_queue command_queue =
              program_.scope->GetCLScpoe()->CommandQueue();
Y
yangfei 已提交
803

804 805 806
          const TensorDesc &desc = var_desc->Tensor_desc();
          //          DDim ddim = make_ddim(desc.Dims());
          DDim ddim = cl_image->dims();
807
          DLOG << var_desc->Name();
L
liuruilong 已提交
808
          cl_image->InitEmptyImage(context, command_queue, ddim);
809
        }
Y
yangfei 已提交
810 811 812 813
      }
    }
  }
}
814

Y
yangfei 已提交
815
template <>
816
void Executor<GPU_CL, float>::InitCombineMemory() {
xiebaiyuan's avatar
xiebaiyuan 已提交
817 818
  DLOG << "CL InitCombineMemory---- "
       << "config_.load_when_predict: " << config_.load_when_predict;
Y
yangfei 已提交
819 820
  char *origin_data = nullptr;
  bool self_alloc = false;
Y
yangfei 已提交
821 822
  if (program_.combined_params_buf && program_.combined_params_len) {
    LOG(kLOG_INFO) << "use outter memory";
823
    origin_data = reinterpret_cast<char *>(program_.combined_params_buf);
Y
yangfei 已提交
824 825
  } else {
    LOG(kLOG_INFO) << " begin init combine memory";
Y
yangfei 已提交
826
    self_alloc = true;
L
liuruilong 已提交
827
    origin_data = ReadFileToBuff(program_.para_path);
Y
yangfei 已提交
828 829
  }
  PADDLE_MOBILE_ENFORCE(origin_data != nullptr, "origin_data==nullptr!!!");
830
  float *data = reinterpret_cast<float *>(origin_data);
Y
yangfei 已提交
831

832
  for (const auto &block : program_desc_->Blocks()) {
Y
yangfei 已提交
833 834 835
    for (const auto &var_desc : block->Vars()) {
      auto var = program_.scope->Var(var_desc->Name());
      if (var_desc->Persistable()) {
L
liuruilong 已提交
836
        CLImage *cl_image = nullptr;
Y
yangfei 已提交
837
        if (var_desc->Name() == "feed" || var_desc->Name() == "fetch") {
H
hjchen2 已提交
838
          var->template GetMutable<framework::LoDTensorArray>();
Y
yangfei 已提交
839
          continue;
L
liuruilong 已提交
840
        } else {
841
          cl_image = var->template GetMutable<CLImage>();
Y
yangfei 已提交
842 843 844 845
        }

        cl_context context = program_.scope->GetCLScpoe()->Context();

846 847
        const TensorDesc &desc = var_desc->Tensor_desc();
        DDim ddim = make_ddim(desc.Dims());
Y
yangfei 已提交
848 849 850 851 852

        int numel = 1;
        for (int i = 0; i < ddim.size(); i++) {
          numel = numel * ddim[i];
        }
853 854 855
        float *tensorInput = static_cast<float *>(
            paddle_mobile::memory::Alloc(sizeof(float) * numel));
        LoadMemory(*var_desc, tensorInput, &origin_data);
L
liuruilong 已提交
856 857 858 859

        // has not init
        cl_image->SetTensorData(tensorInput, ddim);

860 861
        paddle_mobile::memory::Free(tensorInput);
      } else {
862
        auto cl_image = var->template GetMutable<CLImage>();
Y
yangfei 已提交
863
        cl_context context = program_.scope->GetCLScpoe()->Context();
L
liuruilong 已提交
864 865
        cl_command_queue command_queue =
            program_.scope->GetCLScpoe()->CommandQueue();
866 867 868
        const TensorDesc &desc = var_desc->Tensor_desc();
        DDim ddim = cl_image->dims();
        //  DDim ddim = make_ddim(desc.Dims());
L
liuruilong 已提交
869
        cl_image->InitEmptyImage(context, command_queue, ddim);
Y
yangfei 已提交
870 871 872
      }
    }
  }
Y
yangfei 已提交
873
  if (self_alloc) {
874
    delete data;
Y
yangfei 已提交
875
  }
Y
yangfei 已提交
876
  LOG(kLOG_INFO) << " end init combine memory ";
877
}
Y
yangfei 已提交
878 879 880

#endif

881
template class Executor<CPU, float>;
Y
yangfei 已提交
882

883
template class Executor<FPGA, float>;
W
wangliu 已提交
884

885
template class Executor<GPU_CL, float>;
Y
yangfei 已提交
886

887
template class Executor<GPU_MALI, float>;
Y
yangfei 已提交
888 889

}  // namespace framework
W
wangliu 已提交
890
}  // namespace paddle_mobile