add infer time interface, add auc and acc metric

core/CMakeLists.txt

@@ -26,4 +26,5 @@ endif()
 if (NOT CLIENT_ONLY)
 add_subdirectory(predictor)
 add_subdirectory(general-server)
+add_subdirectory(util)
 endif()

core/general-client/include/general_model.h

@@ -51,6 +51,7 @@ class PredictorClient {
                           const std::string& conf_file);
 
   int create_predictor();
+  int destroy_predictor();
 
   std::vector<std::vector<float>> predict(
       const std::vector<std::vector<float>>& float_feed,

core/general-client/src/general_model.cpp

@@ -83,6 +83,11 @@ void PredictorClient::set_predictor_conf(const std::string &conf_path,
   _predictor_conf = conf_file;
 }
 
+int PredictorClient::destroy_predictor() {
+  _api.thrd_finalize();
+  _api.destroy();
+}
+
 int PredictorClient::create_predictor() {
   if (_api.create(_predictor_path.c_str(), _predictor_conf.c_str()) != 0) {
     LOG(ERROR) << "Predictor Creation Failed";
@@ -101,7 +106,9 @@ std::vector<std::vector<float>> PredictorClient::predict(
   if (fetch_name.size() == 0) {
     return fetch_result;
   }
-  fetch_result.resize(fetch_name.size());
+
+  // we save infer_us at fetch_result[fetch_name.size()]
+  fetch_result.resize(fetch_name.size() + 1);
 
   _api.thrd_clear();
   _predictor = _api.fetch_predictor("general_model");
@@ -179,6 +186,8 @@ std::vector<std::vector<float>> PredictorClient::predict(
             *(const float *)res.insts(0).tensor_array(idx).data(i).c_str();
       }
     }
+    fetch_result[fetch_name.size()].resize(1);
+    fetch_result[fetch_name.size()][0] = res.mean_infer_us();
   }
 
   return fetch_result;

core/general-client/src/pybind_general_model.cpp

@@ -43,6 +43,8 @@ PYBIND11_MODULE(serving_client, m) {
            })
       .def("create_predictor",
            [](PredictorClient &self) { self.create_predictor(); })
+      .def("destroy_predictor",
+           [](PredictorClient &self) { self.destroy_predictor(); })
       .def("predict",
            [](PredictorClient &self,
               const std::vector<std::vector<float>> &float_feed,

core/general-server/CMakeLists.txt

@@ -3,7 +3,7 @@ include(op/CMakeLists.txt)
 include(proto/CMakeLists.txt)
 add_executable(serving ${serving_srcs})
 add_dependencies(serving pdcodegen fluid_cpu_engine pdserving paddle_fluid
-        opencv_imgcodecs cube-api)
+        opencv_imgcodecs cube-api utils)
 if (WITH_GPU)
     add_dependencies(serving fluid_gpu_engine)
 endif()
@@ -23,6 +23,7 @@ target_link_libraries(serving paddle_fluid ${paddle_depend_libs})
 
 target_link_libraries(serving pdserving)
 target_link_libraries(serving cube-api)
+target_link_libraries(serving utils)
 
 target_link_libraries(serving kvdb rocksdb)
 

core/general-server/op/general_infer_op.cpp

@@ -21,12 +21,13 @@
 #include "core/predictor/framework/infer.h"
 #include "core/predictor/framework/memory.h"
 #include "core/predictor/framework/resource.h"
-
+#include "core/util/include/timer.h"
 
 namespace baidu {
 namespace paddle_serving {
 namespace serving {
 
+using baidu::paddle_serving::Timer;
 using baidu::paddle_serving::predictor::MempoolWrapper;
 using baidu::paddle_serving::predictor::general_model::Tensor;
 using baidu::paddle_serving::predictor::general_model::Response;
@@ -54,10 +55,15 @@ int GeneralInferOp::inference() {
   TensorVector *out = butil::get_object<TensorVector>();
   int batch_size = (*in)[0].shape[0];
   // infer
+  Timer timeline;
+  double infer_time = 0.0;
+  timeline.Start();
   if (InferManager::instance().infer(GENERAL_MODEL_NAME, in, out, batch_size)) {
     LOG(ERROR) << "Failed do infer in fluid model: " << GENERAL_MODEL_NAME;
     return -1;
   }
+  timeline.Pause();
+  infer_time = timeline.ElapsedUS();
 
   const Request *req = dynamic_cast<const Request *>(get_request_message());
 
@@ -79,6 +85,8 @@ int GeneralInferOp::inference() {
   // response inst with only fetch_var_names
   Response *res = mutable_data<Response>();
 
+  res->set_mean_infer_us(infer_time);
+
   for (int i = 0; i < batch_size; ++i) {
     FetchInst *fetch_inst = res->add_insts();
     for (auto & idx : fetch_index) {

core/general-server/op/general_reader_op.cpp

@@ -103,12 +103,14 @@ int GeneralReaderOp::inference() {
   VLOG(2) << "print general model config done.";
 
   // check
+  /*
   res->reader_status = conf_check(req, model_config);
   if (res->reader_status != 0) {
     LOG(INFO) << "model conf of server:";
     resource.print_general_model_config(model_config);
     return 0;
   }
+  */
   // package tensor
 
   elem_type.resize(var_num);
@@ -201,13 +203,14 @@ int GeneralReaderOp::inference() {
 
   VLOG(2) << "read data from client success";
   // print request
+  /*
   std::ostringstream oss;
   int64_t *example = reinterpret_cast<int64_t *>((*in)[0].data.data());
   for (int i = 0; i < 10; i++) {
     oss << *(example + i) << " ";
   }
   VLOG(2) << "head element of first feed var : " << oss.str();
-  //
+  */
   return 0;
 }
 DEFINE_OP(GeneralReaderOp);

core/general-server/proto/general_model_service.proto

@@ -40,6 +40,7 @@ message Request {
 
 message Response {
   repeated FetchInst insts = 1;
+  optional float mean_infer_us = 2;
 };
 
 service GeneralModelService {

core/sdk-cpp/proto/general_model_service.proto

@@ -40,6 +40,7 @@ message Request {
 
 message Response {
   repeated FetchInst insts = 1;
+  optional float mean_infer_us = 2;
 };
 
 service GeneralModelService {

core/util/CMakeLists.txt

+include(src/CMakeLists.txt)
+add_library(utils ${util_srcs})

core/util/include/timer.h

+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include <stdlib.h>
+
+
+namespace baidu {
+namespace paddle_serving {
+
+// A Standard Timer implementation for debugging
+class Timer {
+ public:
+  // a timer class for profiling
+  // Reset() will be called during initialization
+  // all timing variables will be set 0 in Reset()
+  Timer() { Reset(); }
+  void Reset();
+  void Start();
+  void Pause();
+  // Resume will get current system time
+  void Resume();
+  int Count();
+  // return elapsed time in us
+  double ElapsedUS();
+  // return elapsed time in ms
+  double ElapsedMS();
+  // return elapsed time in sec
+  double ElapsedSec();
+
+ private:
+  struct timeval _start;
+  struct timeval _now;
+  int _count;
+  int64_t _elapsed;
+  bool _paused;
+
+  // get us difference between start and now
+  int64_t Tickus();
+};
+
+}  // namespace paddle_serving
+}  // namespace baidu

core/util/src/CMakeLists.txt

+FILE(GLOB srcs ${CMAKE_CURRENT_LIST_DIR}/*.cc)
+LIST(APPEND util_srcs ${srcs})

core/util/src/timer.cc

+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <sys/time.h>
+#include "core/util/include/timer.h"
+
+namespace baidu {
+namespace paddle_serving {
+
+void Timer::Reset() {
+  _start.tv_sec = 0;
+  _start.tv_usec = 0;
+
+  _count = 0;
+  _elapsed = 0;
+  _paused = true;
+}
+
+void Timer::Start() {
+  Reset();
+  Resume();
+}
+
+void Timer::Pause() {
+  if (_paused) {
+    return;
+  }
+  _elapsed += Tickus();
+  ++_count;
+  _paused = true;
+}
+
+void Timer::Resume() {
+  gettimeofday(&_start, NULL);
+  _paused = false;
+}
+
+int Timer::Count() { return _count; }
+
+double Timer::ElapsedUS() { return static_cast<double>(_elapsed); }
+
+double Timer::ElapsedMS() { return _elapsed / 1000.0; }
+
+double Timer::ElapsedSec() { return _elapsed / 1000000.0; }
+
+int64_t Timer::Tickus() {
+  gettimeofday(&_now, NULL);
+  return (_now.tv_sec - _start.tv_sec) * 1000 * 1000L +
+         (_now.tv_usec - _start.tv_usec);
+}
+
+}  // namespace paddle_serving
+}  // namespace baidu

python/paddle_serving_client/__init__.py

@@ -118,7 +118,7 @@ class Client(object):
     def get_fetch_names(self):
         return self.fetch_names_
 
-    def predict(self, feed={}, fetch=[]):
+    def predict(self, feed={}, fetch=[], debug=False):
         int_slot = []
         float_slot = []
         int_feed_names = []
@@ -148,6 +148,9 @@ class Client(object):
         for i, name in enumerate(fetch_names):
             result_map[name] = result[i]
 
+        if debug:
+            result_map["infer_time"] = result[-1][0]
+
         return result_map
 
     def batch_predict(self, feed_batch=[], fetch=[]):
@@ -191,3 +194,6 @@ class Client(object):
             result_map_batch.append(result_map)
 
         return result_map_batch
+
+    def release(self):
+        self.client_handle_.destroy_predictor()

python/paddle_serving_client/metric/__init__.py

@@ -12,3 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from auc import auc
+from acc import acc

python/paddle_serving_client/metric/acc.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def acc(prob, label, threshold):
+    # we support prob is the probability for label to be one
+    assert len(prob) == len(label)
+    total = len(prob)
+    right = 0
+    for i in range(len(prob)):
+        if (prob - threshold) * (label - prob) > 0:
+            right += 1
+    return float(right) / total
+
+

python/paddle_serving_client/metric/auc.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 def tied_rank(x):
     """
     Computes the tied rank of elements in x.

python/paddle_serving_client/utils/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import subprocess
+from multiprocessing import Pool
+
+class MultiThreadRunner(object):
+    def __init__(self):
+        pass
+
+    def run(self, thread_func, thread_num, global_resource):
+        p = Pool(thread_num)
+        result_list = []
+        for i in range(thread_num):
+            result_list.append(
+                p.apply_async(thread_func, [i + 1, global_resource]))
+        p.close()
+        p.join()
+        return_result = result_list[0].get()
+        for i in range(1, thread_num, 1):
+            tmp_result = result_list[i].get()
+            for i, item in enumerate(tmp_result):
+                return_result[i].extend(tmp_result[i])
+        return return_result

python/setup.py.client.in

@@ -35,7 +35,8 @@ REQUIRED_PACKAGES = [
 packages=['paddle_serving_client',
           'paddle_serving_client.proto',
           'paddle_serving_client.io',
-	  'paddle_serving_client.metric']
+	  'paddle_serving_client.metric',
+	  'paddle_serving_client.utils']
 package_data={'paddle_serving_client': ['serving_client.so']}
 package_dir={'paddle_serving_client':
              '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client',
@@ -44,7 +45,9 @@ package_dir={'paddle_serving_client':
              'paddle_serving_client.io':
              '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/io',
 	     'paddle_serving_client.metric':
-	     '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/metric'}
+	     '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/metric',
+	     'paddle_serving_client.utils':
+	     '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/utils'}
 
 setup(
     name='paddle-serving-client',