general_model.cpp 11.3 KB
Newer Older
G
guru4elephant 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

G
guru4elephant 已提交
15
#include "core/general-client/include/general_model.h"
M
MRXLT 已提交
16
#include <fstream>
G
guru4elephant 已提交
17 18 19
#include "core/sdk-cpp/builtin_format.pb.h"
#include "core/sdk-cpp/include/common.h"
#include "core/sdk-cpp/include/predictor_sdk.h"
G
guru4elephant 已提交
20 21 22 23 24 25 26 27 28 29

using baidu::paddle_serving::predictor::general_model::Request;
using baidu::paddle_serving::predictor::general_model::Response;
using baidu::paddle_serving::predictor::general_model::Tensor;
using baidu::paddle_serving::predictor::general_model::FeedInst;
using baidu::paddle_serving::predictor::general_model::FetchInst;

namespace baidu {
namespace paddle_serving {
namespace general_model {
30
using configure::GeneralModelConfig;
G
guru4elephant 已提交
31

32 33 34 35 36 37 38 39 40
int PredictorClient::init(const std::string &conf_file) {
  try {
    GeneralModelConfig model_config;
    if (configure::read_proto_conf(conf_file.c_str(),
                                   &model_config) != 0) {
      LOG(ERROR) << "Failed to load general model config"
                 << ", file path: " << conf_file;
      return -1;
    }
41

42 43 44 45 46
    _feed_name_to_idx.clear();
    _fetch_name_to_idx.clear();
    _shape.clear();
    int feed_var_num = model_config.feed_var_size();
    int fetch_var_num = model_config.fetch_var_size();
47 48
    VLOG(2) << "feed var num: " << feed_var_num
            << "fetch_var_num: " << fetch_var_num;
49 50
    for (int i = 0; i < feed_var_num; ++i) {
      _feed_name_to_idx[model_config.feed_var(i).alias_name()] = i;
51 52
      VLOG(2) << "feed alias name: " << model_config.feed_var(i).alias_name()
              << " index: " << i;
53
      std::vector<int> tmp_feed_shape;
54
      VLOG(2) << "feed" << "[" << i << "] shape:";
55 56
      for (int j = 0; j < model_config.feed_var(i).shape_size(); ++j) {
        tmp_feed_shape.push_back(model_config.feed_var(i).shape(j));
57 58
        VLOG(2) << "shape[" << j << "]: "
                << model_config.feed_var(i).shape(j);
59 60
      }
      _type.push_back(model_config.feed_var(i).feed_type());
61 62
      VLOG(2) << "feed" << "[" << i << "] feed type: "
              << model_config.feed_var(i).feed_type();
63
      _shape.push_back(tmp_feed_shape);
G
guru4elephant 已提交
64 65
    }

66 67
    for (int i = 0; i < fetch_var_num; ++i) {
      _fetch_name_to_idx[model_config.fetch_var(i).alias_name()] = i;
68 69
      VLOG(2) << "fetch [" << i << "]" << " alias name: "
              << model_config.fetch_var(i).alias_name();
70 71 72 73 74 75
      _fetch_name_to_var_name[model_config.fetch_var(i).alias_name()] =
          model_config.fetch_var(i).name();
    }
  } catch (std::exception& e) {
    LOG(ERROR) << "Failed load general model config" << e.what();
    return -1;
G
guru4elephant 已提交
76
  }
77
  return 0;
G
guru4elephant 已提交
78 79
}

M
MRXLT 已提交
80 81
void PredictorClient::set_predictor_conf(const std::string &conf_path,
                                         const std::string &conf_file) {
G
guru4elephant 已提交
82 83 84 85
  _predictor_path = conf_path;
  _predictor_conf = conf_file;
}

86 87 88 89 90
int PredictorClient::destroy_predictor() {
  _api.thrd_finalize();
  _api.destroy();
}

G
guru4elephant 已提交
91 92 93 94 95 96 97 98
int PredictorClient::create_predictor_by_desc(const std::string & sdk_desc) {
  if (_api.create(sdk_desc) != 0) {
    LOG(ERROR) << "Predictor Creation Failed";
    return -1;
  }
  _api.thrd_initialize();
}

G
guru4elephant 已提交
99
int PredictorClient::create_predictor() {
G
guru4elephant 已提交
100 101
  VLOG(2) << "Predictor path: " << _predictor_path
          << " predictor file: " << _predictor_conf;
G
guru4elephant 已提交
102 103 104 105 106 107 108
  if (_api.create(_predictor_path.c_str(), _predictor_conf.c_str()) != 0) {
    LOG(ERROR) << "Predictor Creation Failed";
    return -1;
  }
  _api.thrd_initialize();
}

M
MRXLT 已提交
109 110 111 112 113 114 115
std::vector<std::vector<float>> PredictorClient::predict(
    const std::vector<std::vector<float>> &float_feed,
    const std::vector<std::string> &float_feed_name,
    const std::vector<std::vector<int64_t>> &int_feed,
    const std::vector<std::string> &int_feed_name,
    const std::vector<std::string> &fetch_name) {
  std::vector<std::vector<float>> fetch_result;
G
guru4elephant 已提交
116 117 118
  if (fetch_name.size() == 0) {
    return fetch_result;
  }
119 120 121

  // we save infer_us at fetch_result[fetch_name.size()]
  fetch_result.resize(fetch_name.size() + 1);
G
guru4elephant 已提交
122 123 124

  _api.thrd_clear();
  _predictor = _api.fetch_predictor("general_model");
125 126 127
  VLOG(2) << "fetch general model predictor done.";
  VLOG(2) << "float feed name size: " << float_feed_name.size();
  VLOG(2) << "int feed name size: " << int_feed_name.size();
128
  VLOG(2) << "fetch name size: " << fetch_name.size();
G
guru4elephant 已提交
129
  Request req;
130 131 132
  for (auto & name : fetch_name) {
    req.add_fetch_var_names(name);
  }
G
guru4elephant 已提交
133
  std::vector<Tensor *> tensor_vec;
M
MRXLT 已提交
134 135
  FeedInst *inst = req.add_insts();
  for (auto &name : float_feed_name) {
G
guru4elephant 已提交
136 137 138
    tensor_vec.push_back(inst->add_tensor_array());
  }

M
MRXLT 已提交
139
  for (auto &name : int_feed_name) {
G
guru4elephant 已提交
140 141
    tensor_vec.push_back(inst->add_tensor_array());
  }
142
  VLOG(2) << "prepare tensor vec done.";
G
guru4elephant 已提交
143 144

  int vec_idx = 0;
M
MRXLT 已提交
145
  for (auto &name : float_feed_name) {
G
guru4elephant 已提交
146
    int idx = _feed_name_to_idx[name];
M
MRXLT 已提交
147
    Tensor *tensor = tensor_vec[idx];
G
guru4elephant 已提交
148 149 150 151 152
    for (int j = 0; j < _shape[idx].size(); ++j) {
      tensor->add_shape(_shape[idx][j]);
    }
    tensor->set_elem_type(1);
    for (int j = 0; j < float_feed[vec_idx].size(); ++j) {
M
MRXLT 已提交
153 154 155
      tensor->add_data(const_cast<char *>(reinterpret_cast<const char *>(
                           &(float_feed[vec_idx][j]))),
                       sizeof(float));
G
guru4elephant 已提交
156 157 158 159
    }
    vec_idx++;
  }

160 161
  VLOG(2) << "feed float feed var done.";

G
guru4elephant 已提交
162
  vec_idx = 0;
M
MRXLT 已提交
163
  for (auto &name : int_feed_name) {
G
guru4elephant 已提交
164
    int idx = _feed_name_to_idx[name];
M
MRXLT 已提交
165
    Tensor *tensor = tensor_vec[idx];
G
guru4elephant 已提交
166 167 168 169 170
    for (int j = 0; j < _shape[idx].size(); ++j) {
      tensor->add_shape(_shape[idx][j]);
    }
    tensor->set_elem_type(0);
    for (int j = 0; j < int_feed[vec_idx].size(); ++j) {
M
MRXLT 已提交
171 172 173
      tensor->add_data(const_cast<char *>(reinterpret_cast<const char *>(
                           &(int_feed[vec_idx][j]))),
                       sizeof(int64_t));
G
guru4elephant 已提交
174 175 176 177
    }
    vec_idx++;
  }

178 179
  VLOG(2) << "feed int feed var done.";

G
guru4elephant 已提交
180 181 182 183 184 185 186 187
  // std::map<std::string, std::vector<float> > result;
  Response res;

  res.Clear();
  if (_predictor->inference(&req, &res) != 0) {
    LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString();
    exit(-1);
  } else {
M
MRXLT 已提交
188
    for (auto &name : fetch_name) {
G
guru4elephant 已提交
189 190
      int idx = _fetch_name_to_idx[name];
      int len = res.insts(0).tensor_array(idx).data_size();
191 192
      VLOG(2) << "fetch name: " << name;
      VLOG(2) << "tensor data size: " << len;
G
guru4elephant 已提交
193 194
      fetch_result[idx].resize(len);
      for (int i = 0; i < len; ++i) {
M
MRXLT 已提交
195 196
        fetch_result[idx][i] =
            *(const float *)res.insts(0).tensor_array(idx).data(i).c_str();
G
guru4elephant 已提交
197 198
      }
    }
199 200
    fetch_result[fetch_name.size()].resize(1);
    fetch_result[fetch_name.size()][0] = res.mean_infer_us();
G
guru4elephant 已提交
201 202 203 204 205
  }

  return fetch_result;
}

M
MRXLT 已提交
206
std::vector<std::vector<std::vector<float>>> PredictorClient::batch_predict(
M
MRXLT 已提交
207 208 209 210
    const std::vector<std::vector<std::vector<float>>> &float_feed_batch,
    const std::vector<std::string> &float_feed_name,
    const std::vector<std::vector<std::vector<int64_t>>> &int_feed_batch,
    const std::vector<std::string> &int_feed_name,
M
MRXLT 已提交
211 212
    const std::vector<std::string> &fetch_name) {
  int batch_size = std::max(float_feed_batch.size(), int_feed_batch.size());
M
MRXLT 已提交
213 214 215 216 217 218 219 220 221 222 223 224
  std::vector<std::vector<std::vector<float>>> fetch_result_batch;
  if (fetch_name.size() == 0) {
    return fetch_result_batch;
  }
  fetch_result_batch.resize(batch_size);
  int fetch_name_num = fetch_name.size();
  for (int bi = 0; bi < batch_size; bi++) {
    fetch_result_batch[bi].resize(fetch_name_num);
  }

  _api.thrd_clear();
  _predictor = _api.fetch_predictor("general_model");
225 226 227
  VLOG(2) << "fetch general model predictor done.";
  VLOG(2) << "float feed name size: " << float_feed_name.size();
  VLOG(2) << "int feed name size: " << int_feed_name.size();
M
MRXLT 已提交
228 229 230
  Request req;
  //
  for (int bi = 0; bi < batch_size; bi++) {
231
    VLOG(2) << "prepare batch " << bi;
M
MRXLT 已提交
232 233 234 235 236 237 238 239 240 241 242
    std::vector<Tensor *> tensor_vec;
    FeedInst *inst = req.add_insts();
    std::vector<std::vector<float>> float_feed = float_feed_batch[bi];
    std::vector<std::vector<int64_t>> int_feed = int_feed_batch[bi];
    for (auto &name : float_feed_name) {
      tensor_vec.push_back(inst->add_tensor_array());
    }

    for (auto &name : int_feed_name) {
      tensor_vec.push_back(inst->add_tensor_array());
    }
243 244 245
    
    VLOG(2) << "batch [" << bi << "] int_feed_name and float_feed_name"
            << "prepared";
M
MRXLT 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
    int vec_idx = 0;
    for (auto &name : float_feed_name) {
      int idx = _feed_name_to_idx[name];
      Tensor *tensor = tensor_vec[idx];
      for (int j = 0; j < _shape[idx].size(); ++j) {
        tensor->add_shape(_shape[idx][j]);
      }
      tensor->set_elem_type(1);
      for (int j = 0; j < float_feed[vec_idx].size(); ++j) {
        tensor->add_data(const_cast<char *>(reinterpret_cast<const char *>(
                             &(float_feed[vec_idx][j]))),
                         sizeof(float));
      }
      vec_idx++;
    }

262 263
    VLOG(2) << "batch [" << bi << "] " << "float feed value prepared";

M
MRXLT 已提交
264 265 266 267 268 269 270 271
    vec_idx = 0;
    for (auto &name : int_feed_name) {
      int idx = _feed_name_to_idx[name];
      Tensor *tensor = tensor_vec[idx];
      for (int j = 0; j < _shape[idx].size(); ++j) {
        tensor->add_shape(_shape[idx][j]);
      }
      tensor->set_elem_type(0);
M
MRXLT 已提交
272 273
      VLOG(3) << "feed var name " << name << " index " << vec_idx
              << "first data " << int_feed[vec_idx][0];
M
MRXLT 已提交
274 275 276 277 278 279 280
      for (int j = 0; j < int_feed[vec_idx].size(); ++j) {
        tensor->add_data(const_cast<char *>(reinterpret_cast<const char *>(
                             &(int_feed[vec_idx][j]))),
                         sizeof(int64_t));
      }
      vec_idx++;
    }
281 282

    VLOG(2) << "batch [" << bi << "] " << "itn feed value prepared";
M
MRXLT 已提交
283 284 285 286 287 288 289 290 291 292 293 294
  }

  Response res;

  res.Clear();
  if (_predictor->inference(&req, &res) != 0) {
    LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString();
    exit(-1);
  } else {
    for (int bi = 0; bi < batch_size; bi++) {
      for (auto &name : fetch_name) {
        int idx = _fetch_name_to_idx[name];
M
MRXLT 已提交
295
        int len = res.insts(bi).tensor_array(idx).data_size();
296 297
        VLOG(2) << "fetch name: " << name;
        VLOG(2) << "tensor data size: " << len;
M
MRXLT 已提交
298
        fetch_result_batch[bi][idx].resize(len);
299
        VLOG(2)
M
MRXLT 已提交
300 301
            << "fetch name " << name << " index " << idx << " first data "
            << *(const float *)res.insts(bi).tensor_array(idx).data(0).c_str();
M
MRXLT 已提交
302 303
        for (int i = 0; i < len; ++i) {
          fetch_result_batch[bi][idx][i] =
M
MRXLT 已提交
304
              *(const float *)res.insts(bi).tensor_array(idx).data(i).c_str();
M
MRXLT 已提交
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
        }
      }
    }
  }

  return fetch_result_batch;
}

std::vector<std::vector<float>> PredictorClient::predict_with_profile(
    const std::vector<std::vector<float>> &float_feed,
    const std::vector<std::string> &float_feed_name,
    const std::vector<std::vector<int64_t>> &int_feed,
    const std::vector<std::string> &int_feed_name,
    const std::vector<std::string> &fetch_name) {
  std::vector<std::vector<float>> res;
G
guru4elephant 已提交
320 321 322 323 324 325
  return res;
}

}  // namespace general_model
}  // namespace paddle_serving
}  // namespace baidu