Skip to content

S
Serving

PaddlePaddle / Serving
接近 3 年 前同步成功

通知 188
Star 833
Fork 253
S
Serving
You need to sign in or sign up before continuing.
提交 8ed696c4 编写于 作者: D dongdaxiang 提交者： dongdaxiang
浏览文件
操作

make fetch variable's shape consistent with output of inference. support numpy... 

make fetch variable's shape consistent with output of inference. support numpy array input for predict interface
上级 a646202f
隐藏空白更改
内联 并排
Showing with 140 addition and 322 deletion
core/general-client/include/general_model.h
浏览文件 @ 8ed696c4
...@@ -45,13 +45,17 @@ class PredictorRes { ...@@ -45,13 +45,17 @@ class PredictorRes {
~PredictorRes() {} ~PredictorRes() {}
public: public:
const std::vector<std::vector<int64_t>>& get_int64_by_name( const std::vector<int64_t>& get_int64_by_name(const std::string& name) {
const std::string& name) { return _int64_value_map[name];
return _int64_map[name];
} }
const std::vector<std::vector<float>>& get_float_by_name( const std::vector<float>& get_float_by_name(const std::string& name) {
const std::string& name) { return _float_value_map[name];
return _float_map[name]; }
const std::vector<int>& get_shape(const std::string& name) {
return _shape_map[name];
}
const std::vector<int>& get_lod(const std::string& name) {
return _lod_map[name];
} }
void set_variant_tag(const std::string& variant_tag) { void set_variant_tag(const std::string& variant_tag) {
_variant_tag = variant_tag; _variant_tag = variant_tag;
...@@ -59,8 +63,10 @@ class PredictorRes { ...@@ -59,8 +63,10 @@ class PredictorRes {
const std::string& variant_tag() { return _variant_tag; } const std::string& variant_tag() { return _variant_tag; }
public: public:
std::map<std::string, std::vector<std::vector<int64_t>>> _int64_map; std::map<std::string, std::vector<int64_t>> _int64_value_map;
std::map<std::string, std::vector<std::vector<float>>> _float_map; std::map<std::string, std::vector<float>> _float_value_map;
std::map<std::string, std::vector<int>> _shape_map;
std::map<std::string, std::vector<int>> _lod_map;
private: private:
std::string _variant_tag; std::string _variant_tag;
...@@ -81,17 +87,8 @@ class PredictorClient { ...@@ -81,17 +87,8 @@ class PredictorClient {
int create_predictor_by_desc(const std::string& sdk_desc); int create_predictor_by_desc(const std::string& sdk_desc);
int create_predictor(); int create_predictor();
int destroy_predictor();
int predict(const std::vector<std::vector<float>>& float_feed, int destroy_predictor();
const std::vector<std::string>& float_feed_name,
const std::vector<std::vector<int>>& float_shape,
const std::vector<std::vector<int64_t>>& int_feed,
const std::vector<std::string>& int_feed_name,
const std::vector<std::vector<int>>& int_shape,
const std::vector<std::string>& fetch_name,
PredictorRes& predict_res, // NOLINT
const int& pid);
int batch_predict( int batch_predict(
const std::vector<std::vector<std::vector<float>>>& float_feed_batch, const std::vector<std::vector<std::vector<float>>>& float_feed_batch,
......
core/general-client/src/general_model.cpp
浏览文件 @ 8ed696c4
...@@ -132,154 +132,6 @@ int PredictorClient::create_predictor() { ...@@ -132,154 +132,6 @@ int PredictorClient::create_predictor() {
_api.thrd_initialize(); _api.thrd_initialize();
} }
int PredictorClient::predict(const std::vector<std::vector<float>> &float_feed,
const std::vector<std::string> &float_feed_name,
const std::vector<std::vector<int>> &float_shape,
const std::vector<std::vector<int64_t>> &int_feed,
const std::vector<std::string> &int_feed_name,
const std::vector<std::vector<int>> &int_shape,
const std::vector<std::string> &fetch_name,
PredictorRes &predict_res,
const int &pid) { // NOLINT
predict_res._int64_map.clear();
predict_res._float_map.clear();
Timer timeline;
int64_t preprocess_start = timeline.TimeStampUS();
_api.thrd_clear();
std::string variant_tag;
_predictor = _api.fetch_predictor("general_model", &variant_tag);
predict_res.set_variant_tag(variant_tag);
Request req;
for (auto &name : fetch_name) {
req.add_fetch_var_names(name);
}
std::vector<Tensor *> tensor_vec;
FeedInst *inst = req.add_insts();
for (auto &name : float_feed_name) {
tensor_vec.push_back(inst->add_tensor_array());
}
for (auto &name : int_feed_name) {
tensor_vec.push_back(inst->add_tensor_array());
}
int vec_idx = 0;
for (int i = 0; i < float_feed_name.size(); ++i) {
int idx = _feed_name_to_idx[float_feed_name[i]];
Tensor *tensor = tensor_vec[idx];
if (float_shape.size() == 0) {
for (int j = 0; j < _shape[idx].size(); ++j) {
tensor->add_shape(_shape[idx][j]);
}
} else {
for (int j = 0; j < float_shape[i].size(); ++j) {
tensor->add_shape(float_shape[i][j]);
}
}
tensor->set_elem_type(1);
for (int j = 0; j < float_feed[vec_idx].size(); ++j) {
tensor->add_float_data(float_feed[vec_idx][j]);
}
vec_idx++;
}
VLOG(2) << "feed float feed var done.";
vec_idx = 0;
for (int i = 0; i < int_feed_name.size(); ++i) {
int idx = _feed_name_to_idx[int_feed_name[i]];
Tensor *tensor = tensor_vec[idx];
if (int_shape.size() == 0) {
for (int j = 0; j < int_shape[i].size(); ++j) {
tensor->add_shape(int_shape[i][j]);
}
} else {
for (int j = 0; j < _shape[idx].size(); ++j) {
tensor->add_shape(_shape[idx][j]);
}
}
tensor->set_elem_type(0);
for (int j = 0; j < int_feed[vec_idx].size(); ++j) {
tensor->add_int64_data(int_feed[vec_idx][j]);
}
vec_idx++;
}
int64_t preprocess_end = timeline.TimeStampUS();
int64_t client_infer_start = timeline.TimeStampUS();
Response res;
int64_t client_infer_end = 0;
int64_t postprocess_start = 0;
int64_t postprocess_end = 0;
if (FLAGS_profile_client) {
if (FLAGS_profile_server) {
req.set_profile_server(true);
}
}
res.Clear();
if (_predictor->inference(&req, &res) != 0) {
LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString();
return -1;
} else {
VLOG(2) << "predict done.";
client_infer_end = timeline.TimeStampUS();
postprocess_start = client_infer_end;
for (auto &name : fetch_name) {
int idx = _fetch_name_to_idx[name];
VLOG(2) << "fetch name: " << name;
if (_fetch_name_to_type[name] == 0) {
int len = res.insts(0).tensor_array(idx).int64_data_size();
VLOG(2) << "fetch tensor : " << name << " type: int64 len : " << len;
predict_res._int64_map[name].resize(1);
predict_res._int64_map[name][0].resize(len);
for (int i = 0; i < len; ++i) {
predict_res._int64_map[name][0][i] =
res.insts(0).tensor_array(idx).int64_data(i);
}
} else if (_fetch_name_to_type[name] == 1) {
int len = res.insts(0).tensor_array(idx).float_data_size();
VLOG(2) << "fetch tensor : " << name << " type: float32 len : " << len;
predict_res._float_map[name].resize(1);
predict_res._float_map[name][0].resize(len);
for (int i = 0; i < len; ++i) {
predict_res._float_map[name][0][i] =
res.insts(0).tensor_array(idx).float_data(i);
}
}
postprocess_end = timeline.TimeStampUS();
}
}
if (FLAGS_profile_client) {
std::ostringstream oss;
oss << "PROFILE\t"
<< "pid:" << pid << "\t"
<< "prepro_0:" << preprocess_start << " "
<< "prepro_1:" << preprocess_end << " "
<< "client_infer_0:" << client_infer_start << " "
<< "client_infer_1:" << client_infer_end << " ";
if (FLAGS_profile_server) {
int op_num = res.profile_time_size() / 2;
for (int i = 0; i < op_num; ++i) {
oss << "op" << i << "_0:" << res.profile_time(i * 2) << " ";
oss << "op" << i << "_1:" << res.profile_time(i * 2 + 1) << " ";
}
}
oss << "postpro_0:" << postprocess_start << " ";
oss << "postpro_1:" << postprocess_end;
fprintf(stderr, "%s\n", oss.str().c_str());
}
return 0;
}
int PredictorClient::batch_predict( int PredictorClient::batch_predict(
const std::vector<std::vector<std::vector<float>>> &float_feed_batch, const std::vector<std::vector<std::vector<float>>> &float_feed_batch,
const std::vector<std::string> &float_feed_name, const std::vector<std::string> &float_feed_name,
...@@ -292,8 +144,10 @@ int PredictorClient::batch_predict( ...@@ -292,8 +144,10 @@ int PredictorClient::batch_predict(
const int &pid) { const int &pid) {
int batch_size = std::max(float_feed_batch.size(), int_feed_batch.size()); int batch_size = std::max(float_feed_batch.size(), int_feed_batch.size());
predict_res_batch._int64_map.clear(); predict_res_batch._int64_value_map.clear();
predict_res_batch._float_map.clear(); predict_res_batch._float_value_map.clear();
predict_res_batch._shape_map.clear();
predict_res_batch._lod_map.clear();
Timer timeline; Timer timeline;
int64_t preprocess_start = timeline.TimeStampUS(); int64_t preprocess_start = timeline.TimeStampUS();
...@@ -310,7 +164,7 @@ int PredictorClient::batch_predict( ...@@ -310,7 +164,7 @@ int PredictorClient::batch_predict(
for (auto &name : fetch_name) { for (auto &name : fetch_name) {
req.add_fetch_var_names(name); req.add_fetch_var_names(name);
} }
//
for (int bi = 0; bi < batch_size; bi++) { for (int bi = 0; bi < batch_size; bi++) {
VLOG(2) << "prepare batch " << bi; VLOG(2) << "prepare batch " << bi;
std::vector<Tensor *> tensor_vec; std::vector<Tensor *> tensor_vec;
...@@ -325,20 +179,14 @@ int PredictorClient::batch_predict( ...@@ -325,20 +179,14 @@ int PredictorClient::batch_predict(
tensor_vec.push_back(inst->add_tensor_array()); tensor_vec.push_back(inst->add_tensor_array());
} }
VLOG(2) << "batch [" << bi << "] int_feed_name and float_feed_name " VLOG(2) << "batch [" << bi << "] int_feed_name and float_feed_name"
<< "prepared"; << "prepared";
int vec_idx = 0; int vec_idx = 0;
for (int i = 0; i < float_feed_name.size(); ++i) { for (auto &name : float_feed_name) {
int idx = _feed_name_to_idx[float_feed_name[i]]; int idx = _feed_name_to_idx[name];
Tensor *tensor = tensor_vec[idx]; Tensor *tensor = tensor_vec[idx];
if (float_shape.size() == float_feed_name.size()) { for (int j = 0; j < float_shape[vec_idx].size(); ++j) {
for (int j = 0; j < float_shape[i].size(); ++j) { tensor->add_shape(float_shape[vec_idx][j]);
tensor->add_shape(float_shape[i][j]);
}
} else {
for (int j = 0; j < _shape[idx].size(); ++j) {
tensor->add_shape(_shape[idx][j]);
}
} }
tensor->set_elem_type(1); tensor->set_elem_type(1);
for (int j = 0; j < float_feed[vec_idx].size(); ++j) { for (int j = 0; j < float_feed[vec_idx].size(); ++j) {
...@@ -351,20 +199,14 @@ int PredictorClient::batch_predict( ...@@ -351,20 +199,14 @@ int PredictorClient::batch_predict(
<< "float feed value prepared"; << "float feed value prepared";
vec_idx = 0; vec_idx = 0;
for (int i = 0; i < int_feed_name.size(); ++i) { for (auto &name : int_feed_name) {
int idx = _feed_name_to_idx[int_feed_name[i]]; int idx = _feed_name_to_idx[name];
Tensor *tensor = tensor_vec[idx]; Tensor *tensor = tensor_vec[idx];
if (int_shape.size() == int_feed_name.size()) { for (int j = 0; j < int_shape[vec_idx].size(); ++j) {
for (int j = 0; j < int_shape[i].size(); ++j) { tensor->add_shape(int_shape[vec_idx][j]);
tensor->add_shape(int_shape[i][j]);
}
} else {
for (int j = 0; j < _shape[idx].size(); ++j) {
tensor->add_shape(_shape[idx][j]);
}
} }
tensor->set_elem_type(0); tensor->set_elem_type(0);
VLOG(3) << "feed var name " << float_feed_name[i] << " index " << vec_idx VLOG(3) << "feed var name " << name << " index " << vec_idx
<< "first data " << int_feed[vec_idx][0]; << "first data " << int_feed[vec_idx][0];
for (int j = 0; j < int_feed[vec_idx].size(); ++j) { for (int j = 0; j < int_feed[vec_idx].size(); ++j) {
tensor->add_int64_data(int_feed[vec_idx][j]); tensor->add_int64_data(int_feed[vec_idx][j]);
...@@ -395,43 +237,47 @@ int PredictorClient::batch_predict( ...@@ -395,43 +237,47 @@ int PredictorClient::batch_predict(
res.Clear(); res.Clear();
if (_predictor->inference(&req, &res) != 0) { if (_predictor->inference(&req, &res) != 0) {
LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString(); LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString();
return -1; exit(-1);
} else { } else {
client_infer_end = timeline.TimeStampUS(); client_infer_end = timeline.TimeStampUS();
postprocess_start = client_infer_end; postprocess_start = client_infer_end;
for (auto &name : fetch_name) { for (auto &name : fetch_name) {
predict_res_batch._int64_map[name].resize(batch_size); int idx = _fetch_name_to_idx[name];
predict_res_batch._float_map[name].resize(batch_size); int shape_size = res.insts(0).tensor_array(idx).shape_size();
predict_res_batch._shape_map[name].resize(shape_size);
for (int i = 0; i < shape_size; ++i) {
predict_res_batch._shape_map[name][i] =
res.insts(0).tensor_array(idx).shape(i);
}
int lod_size = res.insts(0).tensor_array(idx).lod_size();
if (lod_size > 0) {
predict_res_batch._lod_map[name].resize(lod_size);
for (int i = 0; i < lod_size; ++i) {
predict_res_batch._lod_map[name][i] =
res.insts(0).tensor_array(idx).lod(i);
}
}
} }
VLOG(2) << "response batch size " << res.insts_size();
VLOG(2) << "response var nmae " << res.insts(0).tensor_array_size(); for (auto &name : fetch_name) {
for (int bi = 0; bi < batch_size; bi++) { int idx = _fetch_name_to_idx[name];
int idx = 0; if (_fetch_name_to_type[name] == 0) {
for (auto &name : fetch_name) { predict_res_batch._int64_value_map[name].resize(
int len = res.insts(bi).tensor_array(idx).data_size(); res.insts(0).tensor_array(idx).int64_data_size());
if (_fetch_name_to_type[name] == 0) { int size = res.insts(0).tensor_array(idx).int64_data_size();
int len = res.insts(bi).tensor_array(idx).int64_data_size(); for (int i = 0; i < size; ++i) {
VLOG(2) << "fetch tensor : " << name << " type: int64 len : " << len; predict_res_batch._int64_value_map[name][i] =
predict_res_batch._int64_map[name][bi].resize(len); res.insts(0).tensor_array(idx).int64_data(i);
VLOG(2) << "fetch name " << name << " index " << idx << " first data " }
<< res.insts(bi).tensor_array(idx).int64_data(0); } else {
for (int i = 0; i < len; ++i) { predict_res_batch._float_value_map[name].resize(
predict_res_batch._int64_map[name][bi][i] = res.insts(0).tensor_array(idx).float_data_size());
res.insts(bi).tensor_array(idx).int64_data(i); int size = res.insts(0).tensor_array(idx).float_data_size();
} for (int i = 0; i < size; ++i) {
} else if (_fetch_name_to_type[name] == 1) { predict_res_batch._float_value_map[name][i] =
int len = res.insts(bi).tensor_array(idx).float_data_size(); res.insts(0).tensor_array(idx).float_data(i);
VLOG(2) << "fetch tensor : " << name
<< " type: float32 len : " << len;
predict_res_batch._float_map[name][bi].resize(len);
VLOG(2) << "fetch name " << name << " index " << idx << " first data "
<< res.insts(bi).tensor_array(idx).float_data(0);
for (int i = 0; i < len; ++i) {
predict_res_batch._float_map[name][bi][i] =
res.insts(bi).tensor_array(idx).float_data(i);
}
} }
idx += 1;
} }
} }
postprocess_end = timeline.TimeStampUS(); postprocess_end = timeline.TimeStampUS();
......
core/general-client/src/pybind_general_model.cpp
浏览文件 @ 8ed696c4
...@@ -40,6 +40,16 @@ PYBIND11_MODULE(serving_client, m) { ...@@ -40,6 +40,16 @@ PYBIND11_MODULE(serving_client, m) {
return self.get_float_by_name(name); return self.get_float_by_name(name);
}, },
py::return_value_policy::reference) py::return_value_policy::reference)
.def("get_shape",
[](PredictorRes &self, std::string &name) {
return self.get_shape(name);
},
py::return_value_policy::reference)
.def("get_lod",
[](PredictorRes &self, std::string &name) {
return self.get_lod(name);
},
py::return_value_policy::reference)
.def("variant_tag", .def("variant_tag",
[](PredictorRes &self) { return self.variant_tag(); }); [](PredictorRes &self) { return self.variant_tag(); });
...@@ -67,27 +77,6 @@ PYBIND11_MODULE(serving_client, m) { ...@@ -67,27 +77,6 @@ PYBIND11_MODULE(serving_client, m) {
[](PredictorClient &self) { self.create_predictor(); }) [](PredictorClient &self) { self.create_predictor(); })
.def("destroy_predictor", .def("destroy_predictor",
[](PredictorClient &self) { self.destroy_predictor(); }) [](PredictorClient &self) { self.destroy_predictor(); })
.def("predict",
[](PredictorClient &self,
const std::vector<std::vector<float>> &float_feed,
const std::vector<std::string> &float_feed_name,
const std::vector<std::vector<int>> &float_shape,
const std::vector<std::vector<int64_t>> &int_feed,
const std::vector<std::string> &int_feed_name,
const std::vector<std::vector<int>> &int_shape,
const std::vector<std::string> &fetch_name,
PredictorRes &predict_res,
const int &pid) {
return self.predict(float_feed,
float_feed_name,
float_shape,
int_feed,
int_feed_name,
int_shape,
fetch_name,
predict_res,
pid);
})
.def("batch_predict", .def("batch_predict",
[](PredictorClient &self, [](PredictorClient &self,
const std::vector<std::vector<std::vector<float>>> const std::vector<std::vector<std::vector<float>>>
......
core/general-server/op/general_response_op.cpp
浏览文件 @ 8ed696c4
...@@ -73,22 +73,21 @@ int GeneralResponseOp::inference() { ...@@ -73,22 +73,21 @@ int GeneralResponseOp::inference() {
// response inst with only fetch_var_names // response inst with only fetch_var_names
Response *res = mutable_data<Response>(); Response *res = mutable_data<Response>();
FetchInst *fetch_inst = res->add_insts();
for (int i = 0; i < batch_size; ++i) { for (auto &idx : fetch_index) {
FetchInst *fetch_inst = res->add_insts(); Tensor *tensor = fetch_inst->add_tensor_array();
for (auto &idx : fetch_index) { tensor->set_elem_type(1);
Tensor *tensor = fetch_inst->add_tensor_array(); if (model_config->_is_lod_fetch[idx]) {
// currently only response float tensor or lod_tensor VLOG(2) << "out[" << idx << "] is lod_tensor";
tensor->set_elem_type(1); for (int k = 0; k < in->at(idx).shape.size(); ++k) {
if (model_config->_is_lod_fetch[idx]) { VLOG(2) << "shape[" << k << "]: " << in->at(idx).shape[k];
VLOG(2) << "out[" << idx << " is lod_tensor"; tensor->add_shape(in->at(idx).shape[k]);
tensor->add_shape(-1); }
} else { } else {
VLOG(2) << "out[" << idx << "] is tensor"; VLOG(2) << "out[" << idx << "] is tensor";
for (int k = 1; k < in->at(idx).shape.size(); ++k) { for (int k = 0; k < in->at(idx).shape.size(); ++k) {
VLOG(2) << "shape[" << k - 1 << "]: " << in->at(idx).shape[k]; VLOG(2) << "shape[" << k << "]: " << in->at(idx).shape[k];
tensor->add_shape(in->at(idx).shape[k]); tensor->add_shape(in->at(idx).shape[k]);
}
} }
} }
} }
...@@ -96,62 +95,42 @@ int GeneralResponseOp::inference() { ...@@ -96,62 +95,42 @@ int GeneralResponseOp::inference() {
int var_idx = 0; int var_idx = 0;
for (auto &idx : fetch_index) { for (auto &idx : fetch_index) {
int cap = 1; int cap = 1;
for (int j = 1; j < in->at(idx).shape.size(); ++j) { for (int j = 0; j < in->at(idx).shape.size(); ++j) {
cap *= in->at(idx).shape[j]; cap *= in->at(idx).shape[j];
} }
if (in->at(idx).dtype == paddle::PaddleDType::INT64) { if (in->at(idx).dtype == paddle::PaddleDType::INT64) {
int64_t *data_ptr = static_cast<int64_t *>(in->at(idx).data.data()); int64_t *data_ptr = static_cast<int64_t *>(in->at(idx).data.data());
if (model_config->_is_lod_fetch[idx]) { if (model_config->_is_lod_fetch[idx]) {
for (int j = 0; j < batch_size; ++j) { FetchInst *fetch_p = res->mutable_insts(0);
for (int k = in->at(idx).lod[0][j]; k < in->at(idx).lod[0][j + 1]; for (int j = 0; j < in->at(idx).lod[0].size(); ++j) {
k++) { fetch_p->mutable_tensor_array(var_idx)->add_lod(
FetchInst *fetch_p = res->mutable_insts(j); in->at(idx).lod[0][j]);
fetch_p->mutable_tensor_array(var_idx)->add_int64_data(data_ptr[k]); }
} for (int j = 0; j < cap; ++j) {
fetch_p->mutable_tensor_array(var_idx)->add_int64_data(data_ptr[j]);
} }
} else { } else {
int var_size = in->at(idx).shape[0]; FetchInst *fetch_p = res->mutable_insts(0);
if (var_size == batch_size) { for (int j = 0; j < cap; ++j) {
for (int j = 0; j < batch_size; ++j) { fetch_p->mutable_tensor_array(var_idx)->add_float_data(data_ptr[j]);
for (int k = j * cap; k < (j + 1) * cap; ++k) {
FetchInst *fetch_p = res->mutable_insts(j);
fetch_p->mutable_tensor_array(var_idx)->add_int64_data(
data_ptr[k]);
}
}
} else {
for (int j = 0; j < batch_size; ++j) {
FetchInst *fetch_p = res->mutable_insts(j);
fetch_p->mutable_tensor_array(var_idx)->add_int64_data(data_ptr[0]);
}
} }
} }
var_idx++; var_idx++;
} else if (in->at(idx).dtype == paddle::PaddleDType::FLOAT32) { } else if (in->at(idx).dtype == paddle::PaddleDType::FLOAT32) {
float *data_ptr = static_cast<float *>(in->at(idx).data.data()); float *data_ptr = static_cast<float *>(in->at(idx).data.data());
if (model_config->_is_lod_fetch[idx]) { if (model_config->_is_lod_fetch[idx]) {
for (int j = 0; j < batch_size; ++j) { FetchInst *fetch_p = res->mutable_insts(0);
for (int k = in->at(idx).lod[0][j]; k < in->at(idx).lod[0][j + 1]; for (int j = 0; j < in->at(idx).lod[0].size(); ++j) {
k++) { fetch_p->mutable_tensor_array(var_idx)->add_lod(
FetchInst *fetch_p = res->mutable_insts(j); in->at(idx).lod[0][j]);
fetch_p->mutable_tensor_array(var_idx)->add_float_data(data_ptr[k]); }
} for (int j = 0; j < cap; ++j) {
fetch_p->mutable_tensor_array(var_idx)->add_float_data(data_ptr[j]);
} }
} else { } else {
int var_size = in->at(idx).shape[0]; FetchInst *fetch_p = res->mutable_insts(0);
if (var_size == batch_size) { for (int j = 0; j < cap; ++j) {
for (int j = 0; j < batch_size; ++j) { fetch_p->mutable_tensor_array(var_idx)->add_float_data(data_ptr[j]);
for (int k = j * cap; k < (j + 1) * cap; ++k) {
FetchInst *fetch_p = res->mutable_insts(j);
fetch_p->mutable_tensor_array(var_idx)->add_float_data(
data_ptr[k]);
}
}
} else {
for (int j = 0; j < batch_size; ++j) {
FetchInst *fetch_p = res->mutable_insts(j);
fetch_p->mutable_tensor_array(var_idx)->add_float_data(data_ptr[0]);
}
} }
} }
var_idx++; var_idx++;
......
core/general-server/proto/general_model_service.proto
浏览文件 @ 8ed696c4
...@@ -26,6 +26,7 @@ message Tensor { ...@@ -26,6 +26,7 @@ message Tensor {
repeated float float_data = 4; repeated float float_data = 4;
optional int32 elem_type = 5; optional int32 elem_type = 5;
repeated int32 shape = 6; repeated int32 shape = 6;
repeated int32 lod = 7; // only for fetch tensor currently
}; };
message FeedInst { repeated Tensor tensor_array = 1; }; message FeedInst { repeated Tensor tensor_array = 1; };
......
core/sdk-cpp/proto/general_model_service.proto
浏览文件 @ 8ed696c4
...@@ -26,6 +26,7 @@ message Tensor { ...@@ -26,6 +26,7 @@ message Tensor {
repeated float float_data = 4; repeated float float_data = 4;
optional int32 elem_type = 5; optional int32 elem_type = 5;
repeated int32 shape = 6; repeated int32 shape = 6;
repeated int32 lod = 7; // only for fetch tensor currently
}; };
message FeedInst { repeated Tensor tensor_array = 1; }; message FeedInst { repeated Tensor tensor_array = 1; };
......
python/paddle_serving_client/__init__.py
浏览文件 @ 8ed696c4
...@@ -121,6 +121,7 @@ class Client(object): ...@@ -121,6 +121,7 @@ class Client(object):
self.fetch_names_to_idx_ = {} self.fetch_names_to_idx_ = {}
self.lod_tensor_set = set() self.lod_tensor_set = set()
self.feed_tensor_len = {} self.feed_tensor_len = {}
for i, var in enumerate(model_conf.feed_var): for i, var in enumerate(model_conf.feed_var):
self.feed_names_to_idx_[var.alias_name] = i self.feed_names_to_idx_[var.alias_name] = i
self.feed_types_[var.alias_name] = var.feed_type self.feed_types_[var.alias_name] = var.feed_type
...@@ -133,11 +134,11 @@ class Client(object): ...@@ -133,11 +134,11 @@ class Client(object):
for dim in self.feed_shapes_[var.alias_name]: for dim in self.feed_shapes_[var.alias_name]:
counter *= dim counter *= dim
self.feed_tensor_len[var.alias_name] = counter self.feed_tensor_len[var.alias_name] = counter
for i, var in enumerate(model_conf.fetch_var): for i, var in enumerate(model_conf.fetch_var):
self.fetch_names_to_idx_[var.alias_name] = i self.fetch_names_to_idx_[var.alias_name] = i
self.fetch_names_to_type_[var.alias_name] = var.fetch_type self.fetch_names_to_type_[var.alias_name] = var.fetch_type
if var.is_lod_tensor:
self.lod_tensor_set.add(var.alias_name)
return return
def add_variant(self, tag, cluster, variant_weight): def add_variant(self, tag, cluster, variant_weight):
...@@ -164,7 +165,6 @@ class Client(object): ...@@ -164,7 +165,6 @@ class Client(object):
"parameter endpoints({}) will not take effect, because you use the add_variant function.". "parameter endpoints({}) will not take effect, because you use the add_variant function.".
format(endpoints)) format(endpoints))
sdk_desc = self.predictor_sdk_.gen_desc() sdk_desc = self.predictor_sdk_.gen_desc()
print(sdk_desc)
self.client_handle_.create_predictor_by_desc(sdk_desc.SerializeToString( self.client_handle_.create_predictor_by_desc(sdk_desc.SerializeToString(
)) ))
...@@ -228,23 +228,25 @@ class Client(object): ...@@ -228,23 +228,25 @@ class Client(object):
for key in feed_i: for key in feed_i:
if key not in self.feed_names_: if key not in self.feed_names_:
raise ValueError("Wrong feed name: {}.".format(key)) raise ValueError("Wrong feed name: {}.".format(key))
self.shape_check(feed_i, key) if not isinstance(feed_i[key], np.ndarray):
self.shape_check(feed_i, key)
if self.feed_types_[key] == int_type: if self.feed_types_[key] == int_type:
if i == 0: if i == 0:
int_feed_names.append(key) int_feed_names.append(key)
if isinstance(feed_i[key], np.ndarray): if isinstance(feed_i[key], np.ndarray):
int_shape.append(feed_i[key].shape) int_shape.append(list(feed_i[key].shape))
if isinstance(feed_i[key], np.ndarray): if isinstance(feed_i[key], np.ndarray):
int_slot.append(feed_i[key].tolist()) int_slot.append(np.reshape(feed_i[key], (-1)).tolist())
else: else:
int_slot.append(feed_i[key]) int_slot.append(feed_i[key])
elif self.feed_types_[key] == float_type: elif self.feed_types_[key] == float_type:
if i == 0: if i == 0:
float_feed_names.append(key) float_feed_names.append(key)
if isinstance(feed_i[key], np.ndarray): if isinstance(feed_i[key], np.ndarray):
float_shape.append(feed_i[key].shape) float_shape.append(list(feed_i[key].shape))
if isinstance(feed_i[key], np.ndarray): if isinstance(feed_i[key], np.ndarray):
float_slot.append(feed_i[key].tolist()) float_slot.append(
np.reshape(feed_i[key], (-1)).tolist())
else: else:
float_slot.append(feed_i[key]) float_slot.append(feed_i[key])
int_slot_batch.append(int_slot) int_slot_batch.append(int_slot)
...@@ -252,31 +254,34 @@ class Client(object): ...@@ -252,31 +254,34 @@ class Client(object):
result_batch = self.result_handle_ result_batch = self.result_handle_
res = self.client_handle_.batch_predict( res = self.client_handle_.batch_predict(
float_slot_batch, float_feed_names, int_slot_batch, int_feed_names, float_slot_batch, float_feed_names, float_shape, int_slot_batch,
fetch_names, result_batch, self.pid) int_feed_names, int_shape, fetch_names, result_batch, self.pid)
if res == -1: if res == -1:
return None return None
result_map_batch = [] result_map_batch = []
result_map = {} result_map = {}
# result map needs to be a numpy array
for i, name in enumerate(fetch_names): for i, name in enumerate(fetch_names):
if self.fetch_names_to_type_[name] == int_type: if self.fetch_names_to_type_[name] == int_type:
result_map[name] = result_batch.get_int64_by_name(name) result_map[name] = result_batch.get_int64_by_name(name)
shape = result_batch.get_shape(name)
result_map[name] = np.array(result_map[name])
result_map[name].shape = shape
if name in self.lod_tensor_set:
result_map["{}.lod".format(name)] = result_batch.get_lod(
name)
elif self.fetch_names_to_type_[name] == float_type: elif self.fetch_names_to_type_[name] == float_type:
result_map[name] = result_batch.get_float_by_name(name) result_map[name] = result_batch.get_float_by_name(name)
for i in range(batch_size): shape = result_batch.get_shape(name)
single_result = {} result_map[name] = np.array(result_map[name])
for key in result_map: result_map[name].shape = shape
single_result[key] = result_map[key][i] if name in self.lod_tensor_set:
result_map_batch.append(single_result) result_map["{}.lod".format(name)] = result_batch.get_lod(
name)
if batch_size == 1:
return [result_map_batch[0], self.result_handle_.variant_tag() return result_map
] if need_variant_tag else result_map_batch[0]
else:
return [result_map_batch, self.result_handle_.variant_tag()
] if need_variant_tag else result_map_batch
def release(self): def release(self):
self.client_handle_.destroy_predictor() self.client_handle_.destroy_predictor()
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或