add general-client

general-client/CMakeLists.txt

+if(CLIENT_ONLY)
+add_subdirectory(pybind11)
+pybind11_add_module(serving_client src/general_model.cpp src/pybind_general_model.cpp)
+target_link_libraries(serving_client -Wl,--whole-archive sdk-cpp pybind python -Wl,--no-whole-archive -lpthread -lcrypto -lm -lrt -lssl -ldl -lz)
+endif()

general-client/README.md

+# install
+pip install paddle-serving-client
+# quick-start example
+``` python
+# train text classification problem with paddle
+# save serving model
+```
+from serving_client import serving_io
+
+serving_io.save_serving_model()
+
+
+``` python
+# load serving model configuration and start service
+```
+
+``` python
+# load serving model configuration in client and do prediction
+```
+
+# design
+paddle sering as remote executor run
+
+# high level features
+C++ client

general-client/inference.conf

+2 3
+words 1 -1
+label 1 1
+cost mean_0.tmp_0
+acc accuracy_0.tmp_0
+prediction fc_1.tmp_2

general-client/predictor.conf

+default_variant_conf {
+  tag: "default"
+  connection_conf {
+    connect_timeout_ms: 2000
+    rpc_timeout_ms: 20000
+    connect_retry_count: 2
+    max_connection_per_host: 100
+    hedge_request_timeout_ms: -1
+    hedge_fetch_retry_count: 2
+    connection_type: "pooled"
+  }
+  naming_conf {
+    cluster_filter_strategy: "Default"
+    load_balance_strategy: "la"
+  }
+  rpc_parameter {
+    compress_type: 0
+    package_size: 20
+    protocol: "baidu_std"
+    max_channel_per_request: 3
+  }
+}
+
+predictors {
+  name: "general_model"
+  service_name: "baidu.paddle_serving.predictor.general_model.GeneralModelService"
+  endpoint_router: "WeightedRandomRender"
+  weighted_random_render_conf {
+    variant_weight_list: "50"
+  }
+  variants {
+    tag: "var1"
+    naming_conf {
+      cluster: "list://127.0.0.1:9292"
+    }
+  }
+}

general-client/src/general_model.cpp

+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include "general_model.h"
+#include "sdk-cpp/builtin_format.pb.h"
+#include "sdk-cpp/include/common.h"
+#include "sdk-cpp/include/predictor_sdk.h"
+
+using baidu::paddle_serving::predictor::general_model::Request;
+using baidu::paddle_serving::predictor::general_model::Response;
+using baidu::paddle_serving::predictor::general_model::Tensor;
+using baidu::paddle_serving::predictor::general_model::FeedInst;
+using baidu::paddle_serving::predictor::general_model::FetchInst;
+
+namespace baidu {
+namespace paddle_serving {
+namespace general_model {
+
+void PredictorClient::init(const std::string & conf_file) {
+  _conf_file = conf_file;
+  std::ifstream fin(conf_file);
+  if (!fin) {
+    LOG(ERROR) << "Your inference conf file can not be found";
+    exit(-1);
+  }
+  _feed_name_to_idx.clear();
+  _fetch_name_to_idx.clear();
+  _shape.clear();
+  int feed_var_num = 0;
+  int fetch_var_num = 0;
+  fin >> feed_var_num >> fetch_var_num;
+  std::string name;
+  std::string fetch_var_name;
+  int shape_num = 0;
+  int dim = 0;
+  for (int i = 0; i < feed_var_num; ++i) {
+    fin >> name;
+    _feed_name_to_idx[name] = i;
+    fin >> shape_num;
+    std::vector<int> tmp_feed_shape;
+    for (int j = 0; j < shape_num; ++j) {
+      fin >> dim;
+      tmp_feed_shape.push_back(dim);
+    }
+    _shape.push_back(tmp_feed_shape);
+  }
+
+  for (int i = 0; i < fetch_var_num; ++i) {
+    fin >> name;
+    fin >> fetch_var_name;
+    _fetch_name_to_idx[name] = i;
+    _fetch_name_to_var_name[name] = fetch_var_name;
+  }
+}
+
+void PredictorClient::set_predictor_conf(
+    const std::string & conf_path,
+    const std::string & conf_file) {
+  _predictor_path = conf_path;
+  _predictor_conf = conf_file;
+}
+
+int PredictorClient::create_predictor() {
+  if (_api.create(_predictor_path.c_str(), _predictor_conf.c_str()) != 0) {
+    LOG(ERROR) << "Predictor Creation Failed";
+    return -1;
+  }
+  _api.thrd_initialize();
+}
+
+std::vector<std::vector<float> > PredictorClient::predict(
+    const std::vector<std::vector<float> > & float_feed,
+    const std::vector<std::string> & float_feed_name,
+    const std::vector<std::vector<int64_t> > & int_feed,
+    const std::vector<std::string> & int_feed_name,
+    const std::vector<std::string> & fetch_name) {
+
+  std::vector<std::vector<float> > fetch_result;
+  if (fetch_name.size() == 0) {
+    return fetch_result;
+  }
+  fetch_result.resize(fetch_name.size());
+
+  _api.thrd_clear();
+  _predictor = _api.fetch_predictor("general_model");
+  Request req;
+  std::vector<Tensor *> tensor_vec;
+  FeedInst * inst = req.add_insts();
+  for (auto & name : float_feed_name) {
+    tensor_vec.push_back(inst->add_tensor_array());
+  }
+
+  for (auto & name : int_feed_name) {
+    tensor_vec.push_back(inst->add_tensor_array());
+  }
+
+  int vec_idx = 0;
+  for (auto & name : float_feed_name) {
+    int idx = _feed_name_to_idx[name];
+    Tensor * tensor = tensor_vec[idx];
+    for (int j = 0; j < _shape[idx].size(); ++j) {
+      tensor->add_shape(_shape[idx][j]);
+    }
+    tensor->set_elem_type(1);
+    for (int j = 0; j < float_feed[vec_idx].size(); ++j) {
+      tensor->add_data(
+          (char *)(&(float_feed[vec_idx][j])), sizeof(float));
+    }
+    vec_idx++;
+  }
+
+  vec_idx = 0;
+  for (auto & name : int_feed_name) {
+    int idx = _feed_name_to_idx[name];
+    Tensor * tensor = tensor_vec[idx];
+    for (int j = 0; j < _shape[idx].size(); ++j) {
+      tensor->add_shape(_shape[idx][j]);
+    }
+    tensor->set_elem_type(0);
+    for (int j = 0; j < int_feed[vec_idx].size(); ++j) {
+      tensor->add_data(
+          (char *)(&(int_feed[vec_idx][j])), sizeof(int64_t));
+    }
+    vec_idx++;
+  }
+
+  // std::map<std::string, std::vector<float> > result;
+  Response res;
+
+  res.Clear();
+  if (_predictor->inference(&req, &res) != 0) {
+    LOG(ERROR) << "failed call predictor with req: " << req.ShortDebugString();
+    exit(-1);
+  } else {
+    for (auto & name : fetch_name) {
+      int idx = _fetch_name_to_idx[name];
+      int len = res.insts(0).tensor_array(idx).data_size();
+      VLOG(3) << "fetch name: " << name;
+      VLOG(3) << "tensor data size: " << len;
+      fetch_result[idx].resize(len);
+      for (int i = 0; i < len; ++i) {
+        /*
+        (*fetch_result)[name][i] = *(const float *)
+                    res.insts(0).tensor_array(idx).data(i).c_str();
+        VLOG(3) << *(const float *)
+            res.insts(0).tensor_array(idx).data(i).c_str();
+        fetch_result[name][i] = *(const float *)
+                    res.insts(0).tensor_array(idx).data(i).c_str();
+        */
+        fetch_result[idx][i] = *(const float *)
+                    res.insts(0).tensor_array(idx).data(i).c_str();
+      }
+    }
+  }
+
+  return fetch_result;
+}
+
+std::vector<std::vector<float> > PredictorClient::predict_with_profile(
+    const std::vector<std::vector<float> > & float_feed,
+    const std::vector<std::string> & float_feed_name,
+    const std::vector<std::vector<int64_t> > & int_feed,
+    const std::vector<std::string> & int_feed_name,
+    const std::vector<std::string> & fetch_name) {
+  std::vector<std::vector<float> > res;
+  return res;
+}
+
+}  // namespace general_model
+}  // namespace paddle_serving
+}  // namespace baidu

general-client/src/general_model.h

+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <fstream>
+#include <string>
+#include <vector>
+#include <map>
+
+#include "sdk-cpp/builtin_format.pb.h"
+#include "sdk-cpp/general_model_service.pb.h"
+#include "sdk-cpp/include/common.h"
+#include "sdk-cpp/include/predictor_sdk.h"
+
+using baidu::paddle_serving::sdk_cpp::Predictor;
+using baidu::paddle_serving::sdk_cpp::PredictorApi;
+
+// given some input data, pack into pb, and send request
+namespace baidu {
+namespace paddle_serving {
+namespace general_model {
+
+typedef std::map<std::string, std::vector<float>> FetchedMap;
+
+typedef std::map<std::string, std::vector<std::vector<float> > >
+    BatchFetchedMap;
+
+class PredictorClient {
+ public:
+  PredictorClient() {}
+  ~PredictorClient() {}
+
+  void init(const std::string & client_conf);
+
+  void set_predictor_conf(
+      const std::string& conf_path,
+      const std::string& conf_file);
+
+  int create_predictor();
+
+  std::vector<std::vector<float> > predict(
+      const std::vector<std::vector<float> > & float_feed,
+      const std::vector<std::string> & float_feed_name,
+      const std::vector<std::vector<int64_t> > & int_feed,
+      const std::vector<std::string> & int_feed_name,
+      const std::vector<std::string> & fetch_name);
+
+  std::vector<std::vector<float> > predict_with_profile(
+      const std::vector<std::vector<float> > & float_feed,
+      const std::vector<std::string> & float_feed_name,
+      const std::vector<std::vector<int64_t> > & int_feed,
+      const std::vector<std::string> & int_feed_name,
+      const std::vector<std::string> & fetch_name);
+
+ private:
+  PredictorApi _api;
+  Predictor * _predictor;
+  std::string _predictor_conf;
+  std::string _predictor_path;
+  std::string _conf_file;
+  std::map<std::string, int> _feed_name_to_idx;
+  std::map<std::string, int> _fetch_name_to_idx;
+  std::map<std::string, std::string> _fetch_name_to_var_name;
+  std::vector<std::vector<int> > _shape;
+};
+
+}  // namespace general_model
+}  // namespace paddle_serving
+}  // namespace baidu
+
+/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */

general-client/src/general_model_main.cpp

+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include <vector>
+
+#include "general_model.h"
+
+using namespace std;
+
+using baidu::paddle_serving::general_model::PredictorClient;
+using baidu::paddle_serving::general_model::FetchedMap;
+
+int main(int argc, char * argv[]) {
+  PredictorClient * client = new PredictorClient();
+  client->init("inference.conf");
+  client->set_predictor_conf("./", "predictor.conf");
+  client->create_predictor();
+  std::vector<std::vector<float> > float_feed;
+  std::vector<std::vector<int64_t> > int_feed;
+  std::vector<std::string> float_feed_name;
+  std::vector<std::string> int_feed_name = {"words", "label"};
+  std::vector<std::string> fetch_name = {"cost", "acc", "prediction"};
+
+  std::string line;
+  int64_t text_id = 0;
+  int64_t label = 0;
+  int text_id_num = 0;
+  int label_num = 0;
+  int line_num = 0;
+  while (cin >> text_id_num) {
+    int_feed.clear();
+    float_feed.clear();
+    std::vector<int64_t> ids;
+    ids.reserve(text_id_num);
+    for (int i = 0; i < text_id_num; ++i) {
+      cin >> text_id;
+      ids.push_back(text_id);
+    }
+    int_feed.push_back(ids);
+    cin >> label_num;
+    cin >> label;
+    int_feed.push_back({label});
+
+    
+    FetchedMap result;
+
+    client->predict(
+        float_feed, float_feed_name,
+        int_feed, int_feed_name, fetch_name,
+        &result);
+
+    cout << label << "\t" << result["prediction"][1] << endl;
+
+    line_num++;
+    if (line_num % 100 == 0) {
+      cerr << "line num: " << line_num << endl;
+    }
+  }
+}

general-client/src/pybind_general_model.cpp

+#include <Python.h>
+#include <pybind11/pybind11.h>
+#include <unordered_map>
+#include "general_model.h"
+
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+using baidu::paddle_serving::general_model::FetchedMap;
+
+namespace baidu {
+namespace paddle_serving {
+namespace general_model {
+
+PYBIND11_MODULE(serving_client, m) {
+  m.doc() = R"pddoc(this is a practice
+       )pddoc";
+  py::class_<PredictorClient>(m, "PredictorClient", py::buffer_protocol())
+      .def(py::init())
+      .def("init",
+           [](PredictorClient &self, const std::string & conf) {
+             self.init(conf);
+           })
+      .def("set_predictor_conf",
+           [](PredictorClient &self, const std::string & conf_path,
+              const std::string & conf_file) {
+             self.set_predictor_conf(conf_path, conf_file);
+           })
+      .def("create_predictor",
+           [](PredictorClient & self) {
+             self.create_predictor();
+           })
+      .def("predict",
+           [](PredictorClient &self,
+              const std::vector<std::vector<float> > & float_feed,
+              const std::vector<std::string> & float_feed_name,
+              const std::vector<std::vector<int64_t> > & int_feed,
+              const std::vector<std::string> & int_feed_name,
+              const std::vector<std::string> & fetch_name) {
+
+             return self.predict(float_feed, float_feed_name,
+                                 int_feed, int_feed_name, fetch_name);
+           });
+}
+
+}  // namespace general_model
+}  // namespace paddle_serving
+}  // namespace baidu