diff --git a/general-client/CMakeLists.txt b/general-client/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4588218b4e72030ad3aa4240c7ce0289494bfb95 --- /dev/null +++ b/general-client/CMakeLists.txt @@ -0,0 +1,5 @@ +if(CLIENT_ONLY) +add_subdirectory(pybind11) +pybind11_add_module(serving_client src/general_model.cpp src/pybind_general_model.cpp) +target_link_libraries(serving_client -Wl,--whole-archive sdk-cpp pybind python -Wl,--no-whole-archive -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) +endif() diff --git a/general-client/README.md b/general-client/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4045f2221b5e3456ad84c34d4d7e401489c6b2e5 --- /dev/null +++ b/general-client/README.md @@ -0,0 +1,25 @@ +# install +pip install paddle-serving-client +# quick-start example +``` python +# train text classification problem with paddle +# save serving model +``` +from serving_client import serving_io + +serving_io.save_serving_model() + + +``` python +# load serving model configuration and start service +``` + +``` python +# load serving model configuration in client and do prediction +``` + +# design +paddle sering as remote executor run + +# high level features +C++ client diff --git a/general-client/inference.conf b/general-client/inference.conf new file mode 100644 index 0000000000000000000000000000000000000000..fe6710ddad0d498b3d1aebfd7efd3368c874d353 --- /dev/null +++ b/general-client/inference.conf @@ -0,0 +1,6 @@ +2 3 +words 1 -1 +label 1 1 +cost mean_0.tmp_0 +acc accuracy_0.tmp_0 +prediction fc_1.tmp_2 diff --git a/general-client/predictor.conf b/general-client/predictor.conf new file mode 100644 index 0000000000000000000000000000000000000000..33d0a84418b24b3ba590f4acfe7dab876c004694 --- /dev/null +++ b/general-client/predictor.conf @@ -0,0 +1,37 @@ +default_variant_conf { + tag: "default" + connection_conf { + connect_timeout_ms: 2000 + rpc_timeout_ms: 20000 + connect_retry_count: 2 + max_connection_per_host: 100 + hedge_request_timeout_ms: -1 + hedge_fetch_retry_count: 2 + connection_type: "pooled" + } + naming_conf { + cluster_filter_strategy: "Default" + load_balance_strategy: "la" + } + rpc_parameter { + compress_type: 0 + package_size: 20 + protocol: "baidu_std" + max_channel_per_request: 3 + } +} + +predictors { + name: "general_model" + service_name: "baidu.paddle_serving.predictor.general_model.GeneralModelService" + endpoint_router: "WeightedRandomRender" + weighted_random_render_conf { + variant_weight_list: "50" + } + variants { + tag: "var1" + naming_conf { + cluster: "list://127.0.0.1:9292" + } + } +} diff --git a/general-client/src/general_model.cpp b/general-client/src/general_model.cpp new file mode 100644 index 0000000000000000000000000000000000000000..378134bb012f05886f20ddad02b395cd3108b08e --- /dev/null +++ b/general-client/src/general_model.cpp @@ -0,0 +1,183 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "general_model.h" +#include "sdk-cpp/builtin_format.pb.h" +#include "sdk-cpp/include/common.h" +#include "sdk-cpp/include/predictor_sdk.h" + +using baidu::paddle_serving::predictor::general_model::Request; +using baidu::paddle_serving::predictor::general_model::Response; +using baidu::paddle_serving::predictor::general_model::Tensor; +using baidu::paddle_serving::predictor::general_model::FeedInst; +using baidu::paddle_serving::predictor::general_model::FetchInst; + +namespace baidu { +namespace paddle_serving { +namespace general_model { + +void PredictorClient::init(const std::string & conf_file) { + _conf_file = conf_file; + std::ifstream fin(conf_file); + if (!fin) { + LOG(ERROR) << "Your inference conf file can not be found"; + exit(-1); + } + _feed_name_to_idx.clear(); + _fetch_name_to_idx.clear(); + _shape.clear(); + int feed_var_num = 0; + int fetch_var_num = 0; + fin >> feed_var_num >> fetch_var_num; + std::string name; + std::string fetch_var_name; + int shape_num = 0; + int dim = 0; + for (int i = 0; i < feed_var_num; ++i) { + fin >> name; + _feed_name_to_idx[name] = i; + fin >> shape_num; + std::vector tmp_feed_shape; + for (int j = 0; j < shape_num; ++j) { + fin >> dim; + tmp_feed_shape.push_back(dim); + } + _shape.push_back(tmp_feed_shape); + } + + for (int i = 0; i < fetch_var_num; ++i) { + fin >> name; + fin >> fetch_var_name; + _fetch_name_to_idx[name] = i; + _fetch_name_to_var_name[name] = fetch_var_name; + } +} + +void PredictorClient::set_predictor_conf( + const std::string & conf_path, + const std::string & conf_file) { + _predictor_path = conf_path; + _predictor_conf = conf_file; +} + +int PredictorClient::create_predictor() { + if (_api.create(_predictor_path.c_str(), _predictor_conf.c_str()) != 0) { + LOG(ERROR) << "Predictor Creation Failed"; + return -1; + } + _api.thrd_initialize(); +} + +std::vector > PredictorClient::predict( + const std::vector > & float_feed, + const std::vector & float_feed_name, + const std::vector > & int_feed, + const std::vector & int_feed_name, + const std::vector & fetch_name) { + + std::vector > fetch_result; + if (fetch_name.size() == 0) { + return fetch_result; + } + fetch_result.resize(fetch_name.size()); + + _api.thrd_clear(); + _predictor = _api.fetch_predictor("general_model"); + Request req; + std::vector tensor_vec; + FeedInst * inst = req.add_insts(); + for (auto & name : float_feed_name) { + tensor_vec.push_back(inst->add_tensor_array()); + } + + for (auto & name : int_feed_name) { + tensor_vec.push_back(inst->add_tensor_array()); + } + + int vec_idx = 0; + for (auto & name : float_feed_name) { + int idx = _feed_name_to_idx[name]; + Tensor * tensor = tensor_vec[idx]; + for (int j = 0; j < _shape[idx].size(); ++j) { + tensor->add_shape(_shape[idx][j]); + } + tensor->set_elem_type(1); + for (int j = 0; j < float_feed[vec_idx].size(); ++j) { + tensor->add_data( + (char *)(&(float_feed[vec_idx][j])), sizeof(float)); + } + vec_idx++; + } + + vec_idx = 0; + for (auto & name : int_feed_name) { + int idx = _feed_name_to_idx[name]; + Tensor * tensor = tensor_vec[idx]; + for (int j = 0; j < _shape[idx].size(); ++j) { + tensor->add_shape(_shape[idx][j]); + } + tensor->set_elem_type(0); + for (int j = 0; j < int_feed[vec_idx].size(); ++j) { + tensor->add_data( + (char *)(&(int_feed[vec_idx][j])), sizeof(int64_t)); + } + vec_idx++; + } + + // std::map > result; + Response res; + + res.Clear(); + if (_predictor->inference(&req, &res) != 0) { + LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString(); + exit(-1); + } else { + for (auto & name : fetch_name) { + int idx = _fetch_name_to_idx[name]; + int len = res.insts(0).tensor_array(idx).data_size(); + VLOG(3) << "fetch name: " << name; + VLOG(3) << "tensor data size: " << len; + fetch_result[idx].resize(len); + for (int i = 0; i < len; ++i) { + /* + (*fetch_result)[name][i] = *(const float *) + res.insts(0).tensor_array(idx).data(i).c_str(); + VLOG(3) << *(const float *) + res.insts(0).tensor_array(idx).data(i).c_str(); + fetch_result[name][i] = *(const float *) + res.insts(0).tensor_array(idx).data(i).c_str(); + */ + fetch_result[idx][i] = *(const float *) + res.insts(0).tensor_array(idx).data(i).c_str(); + } + } + } + + return fetch_result; +} + +std::vector > PredictorClient::predict_with_profile( + const std::vector > & float_feed, + const std::vector & float_feed_name, + const std::vector > & int_feed, + const std::vector & int_feed_name, + const std::vector & fetch_name) { + std::vector > res; + return res; +} + +} // namespace general_model +} // namespace paddle_serving +} // namespace baidu diff --git a/general-client/src/general_model.h b/general-client/src/general_model.h new file mode 100644 index 0000000000000000000000000000000000000000..e959fb630debae1922443ce91feca542764dbb3c --- /dev/null +++ b/general-client/src/general_model.h @@ -0,0 +1,86 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +#include "sdk-cpp/builtin_format.pb.h" +#include "sdk-cpp/general_model_service.pb.h" +#include "sdk-cpp/include/common.h" +#include "sdk-cpp/include/predictor_sdk.h" + +using baidu::paddle_serving::sdk_cpp::Predictor; +using baidu::paddle_serving::sdk_cpp::PredictorApi; + +// given some input data, pack into pb, and send request +namespace baidu { +namespace paddle_serving { +namespace general_model { + +typedef std::map> FetchedMap; + +typedef std::map > > + BatchFetchedMap; + +class PredictorClient { + public: + PredictorClient() {} + ~PredictorClient() {} + + void init(const std::string & client_conf); + + void set_predictor_conf( + const std::string& conf_path, + const std::string& conf_file); + + int create_predictor(); + + std::vector > predict( + const std::vector > & float_feed, + const std::vector & float_feed_name, + const std::vector > & int_feed, + const std::vector & int_feed_name, + const std::vector & fetch_name); + + std::vector > predict_with_profile( + const std::vector > & float_feed, + const std::vector & float_feed_name, + const std::vector > & int_feed, + const std::vector & int_feed_name, + const std::vector & fetch_name); + + private: + PredictorApi _api; + Predictor * _predictor; + std::string _predictor_conf; + std::string _predictor_path; + std::string _conf_file; + std::map _feed_name_to_idx; + std::map _fetch_name_to_idx; + std::map _fetch_name_to_var_name; + std::vector > _shape; +}; + +} // namespace general_model +} // namespace paddle_serving +} // namespace baidu + +/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/general-client/src/general_model_main.cpp b/general-client/src/general_model_main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8bf2e1793c3ac77ddff1fce5de1630b78be88318 --- /dev/null +++ b/general-client/src/general_model_main.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "general_model.h" + +using namespace std; + +using baidu::paddle_serving::general_model::PredictorClient; +using baidu::paddle_serving::general_model::FetchedMap; + +int main(int argc, char * argv[]) { + PredictorClient * client = new PredictorClient(); + client->init("inference.conf"); + client->set_predictor_conf("./", "predictor.conf"); + client->create_predictor(); + std::vector > float_feed; + std::vector > int_feed; + std::vector float_feed_name; + std::vector int_feed_name = {"words", "label"}; + std::vector fetch_name = {"cost", "acc", "prediction"}; + + std::string line; + int64_t text_id = 0; + int64_t label = 0; + int text_id_num = 0; + int label_num = 0; + int line_num = 0; + while (cin >> text_id_num) { + int_feed.clear(); + float_feed.clear(); + std::vector ids; + ids.reserve(text_id_num); + for (int i = 0; i < text_id_num; ++i) { + cin >> text_id; + ids.push_back(text_id); + } + int_feed.push_back(ids); + cin >> label_num; + cin >> label; + int_feed.push_back({label}); + + + FetchedMap result; + + client->predict( + float_feed, float_feed_name, + int_feed, int_feed_name, fetch_name, + &result); + + cout << label << "\t" << result["prediction"][1] << endl; + + line_num++; + if (line_num % 100 == 0) { + cerr << "line num: " << line_num << endl; + } + } +} diff --git a/general-client/src/pybind_general_model.cpp b/general-client/src/pybind_general_model.cpp new file mode 100644 index 0000000000000000000000000000000000000000..991c635ffaacdbda0c8cce36fe6947dd4832f0e7 --- /dev/null +++ b/general-client/src/pybind_general_model.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include "general_model.h" + +#include + +namespace py = pybind11; + +using baidu::paddle_serving::general_model::FetchedMap; + +namespace baidu { +namespace paddle_serving { +namespace general_model { + +PYBIND11_MODULE(serving_client, m) { + m.doc() = R"pddoc(this is a practice + )pddoc"; + py::class_(m, "PredictorClient", py::buffer_protocol()) + .def(py::init()) + .def("init", + [](PredictorClient &self, const std::string & conf) { + self.init(conf); + }) + .def("set_predictor_conf", + [](PredictorClient &self, const std::string & conf_path, + const std::string & conf_file) { + self.set_predictor_conf(conf_path, conf_file); + }) + .def("create_predictor", + [](PredictorClient & self) { + self.create_predictor(); + }) + .def("predict", + [](PredictorClient &self, + const std::vector > & float_feed, + const std::vector & float_feed_name, + const std::vector > & int_feed, + const std::vector & int_feed_name, + const std::vector & fetch_name) { + + return self.predict(float_feed, float_feed_name, + int_feed, int_feed_name, fetch_name); + }); +} + +} // namespace general_model +} // namespace paddle_serving +} // namespace baidu