Merge branch 'develop' of https://github.com/PaddlePaddle/Serving into develop

core/cube/cube-api/src/cube_cli.cpp

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include <gflags/gflags.h>
+#include <algorithm>
 #include <atomic>
 #include <fstream>
 #include <thread>  //NOLINT
@@ -33,7 +34,7 @@ std::atomic<int> g_concurrency(0);
 
 std::vector<std::vector<uint64_t>> time_list;
 std::vector<uint64_t> request_list;
-int turns = 1000000 / FLAGS_batch;
+int turns = 1000;
 
 namespace {
 inline uint64_t time_diff(const struct timeval& start_time,
@@ -94,14 +95,15 @@ int run(int argc, char** argv, int thread_id) {
   uint64_t file_size = key_list.size();
   uint64_t index = 0;
   uint64_t request = 0;
-
   while (g_concurrency.load() >= FLAGS_thread_num) {
   }
   g_concurrency++;
   time_list[thread_id].resize(turns);
-  while (index < file_size) {
+  while (request < turns) {
     // uint64_t key = strtoul(buffer, NULL, 10);
-
+    if (index >= file_size) {
+      index = 0;
+    }
     keys.push_back(key_list[index]);
     index += 1;
     int ret = 0;
@@ -160,7 +162,7 @@ int run_m(int argc, char** argv) {
   uint64_t sum_time = 0;
   uint64_t max_time = 0;
   uint64_t min_time = 1000000;
-  uint64_t request_num = 0;
+  std::vector<uint64_t> all_time_list;
   for (int i = 0; i < thread_num; i++) {
     for (int j = 0; j < request_list[i]; j++) {
       sum_time += time_list[i][j];
@@ -170,20 +172,29 @@ int run_m(int argc, char** argv) {
       if (time_list[i][j] < min_time) {
         min_time = time_list[i][j];
       }
+      all_time_list.push_back(time_list[i][j]);
     }
-    request_num += request_list[i];
   }
+  std::sort(all_time_list.begin(), all_time_list.end());
   uint64_t mean_time = sum_time / (thread_num * turns);
   uint64_t main_time = time_diff(main_start, main_end);
-  LOG(INFO) << "\n"
-            << thread_num << " thread seek cost"
-            << "\navg = " << std::to_string(mean_time)
-            << "\nmax = " << std::to_string(max_time)
-            << "\nmin = " << std::to_string(min_time);
-  LOG(INFO) << "\ntotal_request = " << std::to_string(request_num)
-            << "\nspeed = " << std::to_string(request_num * 1000000 /
-                                              main_time)  // mean_time us
-            << " query per second";
+  uint64_t request_num = turns * thread_num;
+  LOG(INFO)
+      << "\n"
+      << thread_num << " thread seek cost"
+      << "\navg: " << std::to_string(mean_time) << "\n50 percent: "
+      << std::to_string(all_time_list[static_cast<int>(0.5 * request_num)])
+      << "\n80 percent: "
+      << std::to_string(all_time_list[static_cast<int>(0.8 * request_num)])
+      << "\n90 percent: "
+      << std::to_string(all_time_list[static_cast<int>(0.9 * request_num)])
+      << "\n99 percent: "
+      << std::to_string(all_time_list[static_cast<int>(0.99 * request_num)])
+      << "\n99.9 percent: "
+      << std::to_string(all_time_list[static_cast<int>(0.999 * request_num)])
+      << "\ntotal_request: " << std::to_string(request_num) << "\nspeed: "
+      << std::to_string(turns * 1000000 / main_time)  // mean_time us
+      << " query per second";
   return 0;
 }
 

doc/CUBE_QUANT.md

@@ -42,7 +42,7 @@ cd python/examples/criteo_ctr_with_cube
 python local_train.py
 cp ../../../build_server/core/predictor/seq_generator seq_generator
 cp ../../../build_server/output/bin/cube* ./cube/
-sh cube_prepare_quant.sh &
+sh cube_quant_prepare.sh &
 python test_server_quant.py ctr_serving_model_kv &
 python test_client.py ctr_client_conf/serving_client_conf.prototxt ./raw_data
 ```

doc/CUBE_QUANT_CN.md

@@ -42,7 +42,7 @@ cd python/examples/criteo_ctr_with_cube
 python local_train.py
 cp ../../../build_server/core/predictor/seq_generator seq_generator
 cp ../../../build_server/output/bin/cube* ./cube/
-sh cube_prepare_quant.sh &
+sh cube_quant_prepare.sh &
 python test_server_quant.py ctr_serving_model_kv &
 python test_client.py ctr_client_conf/serving_client_conf.prototxt ./raw_data
 ```

doc/NEW_WEB_SERVICE.md

@@ -2,7 +2,7 @@
 
 ([简体中文](NEW_WEB_SERVICE_CN.md)|English)
 
-This document will take the image classification service based on the Imagenet data set as an example to introduce how to develop a new web service. The complete code can be visited at [here](https://github.com/PaddlePaddle/Serving/blob/develop/python/examples/imagenet/image_classification_service.py).
+This document will take the image classification service based on the Imagenet data set as an example to introduce how to develop a new web service. The complete code can be visited at [here](../python/examples/imagenet/resnet50_web_service.py).
 
 ## WebService base class
 

doc/NEW_WEB_SERVICE_CN.md

@@ -2,7 +2,7 @@
 
 (简体中文|[English](NEW_WEB_SERVICE.md))
 
-本文档将以Imagenet图像分类服务为例，来介绍如何开发一个新的Web Service。您可以在[这里](https://github.com/PaddlePaddle/Serving/blob/develop/python/examples/imagenet/image_classification_service.py)查阅完整的代码。
+本文档将以Imagenet图像分类服务为例，来介绍如何开发一个新的Web Service。您可以在[这里](../python/examples/imagenet/resnet50_web_service.py)查阅完整的代码。
 
 ## WebService基类
 

python/examples/criteo_ctr_with_cube/README.md

@@ -27,7 +27,7 @@ mv cube_app/cube* ./cube/
 sh cube_prepare.sh &
 ```
 
-Here, the sparse parameter is loaded by cube sparse parameter indexing service Cube，for more details please read [Cube: Sparse Parameter Indexing Service (Local Mode)](../../../doc/CUBE_LOCAL.md)
+Here, the sparse parameter is loaded by cube sparse parameter indexing service Cube.
 
 ### Start RPC Predictor, the number of serving thread is 4（configurable in test_server.py）
 

python/examples/criteo_ctr_with_cube/README_CN.md

@@ -25,7 +25,7 @@ mv cube_app/cube* ./cube/
 sh cube_prepare.sh &
 ```
 
-此处，模型当中的稀疏参数会被存放在稀疏参数索引服务Cube当中，关于稀疏参数索引服务Cube的介绍，请阅读[稀疏参数索引服务Cube单机版使用指南](../../../doc/CUBE_LOCAL_CN.md)
+此处，模型当中的稀疏参数会被存放在稀疏参数索引服务Cube当中。
 
 ### 启动RPC预测服务，服务端线程数为4（可在test_server.py配置）
 

python/examples/criteo_ctr_with_cube/benchmark_cube.sh

 rm profile_log
 
-wget https://paddle-serving.bj.bcebos.com/unittest/ctr_cube_unittest.tar.gz --no-check-certificate
-tar xf ctr_cube_unittest.tar.gz
+#wget https://paddle-serving.bj.bcebos.com/unittest/ctr_cube_unittest.tar.gz --no-check-certificate
+#tar xf ctr_cube_unittest.tar.gz
 mv models/ctr_client_conf ./
 mv models/ctr_serving_model_kv ./
 mv models/data ./cube/
 
-wget https://paddle-serving.bj.bcebos.com/others/cube_app.tar.gz --no-check-certificate
-tar xf cube_app.tar.gz
+#wget https://paddle-serving.bj.bcebos.com/others/cube_app.tar.gz --no-check-certificate
+#tar xf cube_app.tar.gz
 mv cube_app/cube* ./cube/
 sh cube_prepare.sh &
 
@@ -24,8 +24,7 @@ do
     echo "========================================"
     echo "batch size : $batch_size" >> profile_log
     echo "thread num : $thread_num" >> profile_log
-    tail -n 7 profile | head -n 4 >> profile_log
-    tail -n 2 profile >> profile_log
+    tail -n 8 profile >> profile_log
 
 done
 done

python/examples/imagenet/test_image_reader.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_app.reader.image_reader import String2Image, Base64ToImage, Sequential
+import base64
+
+
+def test_String2Image():
+    with open("./daisy.jpg") as f:
+        img_str = f.read()
+    seq = Sequential([String2Image()])
+    img = seq(img_str)
+    assert (img.shape == (563, 500, 3))
+
+
+def test_Base64ToImage():
+    with open("./daisy.jpg") as f:
+        img_str = f.read()
+    seq = Sequential([Base64ToImage()])
+    img = seq(base64.b64encode(img_str))
+    assert (img.shape == (563, 500, 3))
+
+
+if __name__ == "__main__":
+    test_String2Image()
+    test_Base64ToImage()

python/paddle_serving_app/reader/image_reader.py

@@ -517,6 +517,19 @@ class BGR2RGB(object):
         return self.__class__.__name__ + "()"
 
 
+class String2Image(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, img_buffer):
+        data = np.fromstring(img_buffer, np.uint8)
+        img = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        return img
+
+    def __repr__(self):
+        return self.__class__.__name__ + "()"
+
+
 class File2Image(object):
     def __init__(self):
         pass
@@ -561,7 +574,9 @@ class Base64ToImage(object):
         pass
 
     def __call__(self, img_base64):
-        img = base64.b64decode(img_base64)
+        sample = base64.b64decode(img_base64)
+        data = np.fromstring(sample, np.uint8)
+        img = cv2.imdecode(data, cv2.IMREAD_COLOR)
         return img
 
     def __repr__(self):

python/paddle_serving_app/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving App version string """
-serving_app_version = "0.1.1"
+serving_app_version = "0.1.2"

python/paddle_serving_client/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.1"
-serving_server_version = "0.3.1"
-module_proto_version = "0.3.1"
+serving_client_version = "0.3.2"
+serving_server_version = "0.3.2"
+module_proto_version = "0.3.2"

python/paddle_serving_server/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.1"
-serving_server_version = "0.3.1"
-module_proto_version = "0.3.1"
+serving_client_version = "0.3.2"
+serving_server_version = "0.3.2"
+module_proto_version = "0.3.2"

python/paddle_serving_server/web_service.py

@@ -88,8 +88,8 @@ class WebService(object):
             result = self.postprocess(
                 feed=request.json["feed"], fetch=fetch, fetch_map=fetch_map)
             result = {"result": result}
-        except ValueError:
-            result = {"result": "Request Value Error"}
+        except ValueError as err:
+            result = {"result": err}
         return result
 
     def run_rpc_service(self):

python/paddle_serving_server_gpu/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.1"
-serving_server_version = "0.3.1"
-module_proto_version = "0.3.1"
+serving_client_version = "0.3.2"
+serving_server_version = "0.3.2"
+module_proto_version = "0.3.2"

python/paddle_serving_server_gpu/web_service.py

@@ -133,8 +133,8 @@ class WebService(object):
             result = self.postprocess(
                 feed=request.json["feed"], fetch=fetch, fetch_map=fetch_map)
             result = {"result": result}
-        except ValueError:
-            result = {"result": "Request Value Error"}
+        except ValueError as err:
+            result = {"result": err}
         return result
 
     def run_rpc_service(self):

python/setup.py.client.in

@@ -58,7 +58,7 @@ if '${PACK}' == 'ON':
 
 
 REQUIRED_PACKAGES = [
-    'six >= 1.10.0', 'protobuf >= 3.1.0', 'numpy >= 1.12', 'grpcio >= 1.28.1',
+    'six >= 1.10.0', 'protobuf >= 3.11.0', 'numpy >= 1.12', 'grpcio >= 1.28.1',
     'grpcio-tools >= 1.28.1'
 ]
 

python/setup.py.server.in

@@ -37,7 +37,7 @@ def python_version():
 max_version, mid_version, min_version = python_version()
 
 REQUIRED_PACKAGES = [
-    'six >= 1.10.0', 'protobuf >= 3.1.0', 'grpcio >= 1.28.1', 'grpcio-tools >= 1.28.1',
+    'six >= 1.10.0', 'protobuf >= 3.11.0', 'grpcio >= 1.28.1', 'grpcio-tools >= 1.28.1',
     'paddle_serving_client', 'flask >= 1.1.1', 'paddle_serving_app'
 ]
 

python/setup.py.server_gpu.in

@@ -37,7 +37,7 @@ def python_version():
 max_version, mid_version, min_version = python_version()
 
 REQUIRED_PACKAGES = [
-    'six >= 1.10.0', 'protobuf >= 3.1.0', 'grpcio >= 1.28.1', 'grpcio-tools >= 1.28.1',
+    'six >= 1.10.0', 'protobuf >= 3.11.0', 'grpcio >= 1.28.1', 'grpcio-tools >= 1.28.1',
     'paddle_serving_client', 'flask >= 1.1.1', 'paddle_serving_app'
 ]
 

tools/Dockerfile.gpu

@@ -3,7 +3,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel-centos7 as builder
 FROM nvidia/cuda:9.0-cudnn7-runtime-centos7
 RUN yum -y install wget && \
     yum -y install epel-release && yum -y install patchelf && \
-    yum -y install gcc make python-devel && \
+    yum -y install gcc gcc-c++ make python-devel && \
     yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false && \
     yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false && \
     yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false && \

tools/Dockerfile.gpu.devel

 FROM nvidia/cuda:9.0-cudnn7-devel-centos7
 RUN yum -y install wget >/dev/null \
     && yum -y install gcc gcc-c++ make glibc-static which  \
-    && yum -y install git openssl-devel curl-devel bzip2-devel python-devel
+    && yum -y install git openssl-devel curl-devel bzip2-devel python-devel \
+    && yum -y install libSM-1.2.2-2.el7.x86_64 --setopt=protected_multilib=false \
+    && yum -y install libXrender-0.9.10-1.el7.x86_64 --setopt=protected_multilib=false \
+    && yum -y install libXext-1.3.3-3.el7.x86_64 --setopt=protected_multilib=false
 
 RUN wget https://cmake.org/files/v3.2/cmake-3.2.0-Linux-x86_64.tar.gz >/dev/null \
     && tar xzf cmake-3.2.0-Linux-x86_64.tar.gz \

tools/serving_build.sh

@@ -61,7 +61,7 @@ function build_app() {
                   -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so \
                   -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python \
                   -DAPP=ON ..
-            rerun "make -j2 >/dev/null" 3 # due to some network reasons, compilation may fail
+            rerun "make -j10 >/dev/null" 3 # due to some network reasons, compilation may fail
             pip install -U python/dist/paddle_serving_app* >/dev/null
             ;;
         *)
@@ -84,7 +84,7 @@ function build_client() {
                   -DPYTHON_LIBRARIES=$PYTHONROOT/lib64/libpython2.7.so \
                   -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python \
                   -DCLIENT=ON ..
-            rerun "make -j2 >/dev/null" 3 # due to some network reasons, compilation may fail
+            rerun "make -j10 >/dev/null" 3 # due to some network reasons, compilation may fail
             pip install -U python/dist/paddle_serving_client* >/dev/null
             ;;
         *)
@@ -108,7 +108,7 @@ function build_server() {
                   -DPYTHON_LIBRARIES=$PYTHONROOT/lib64/libpython2.7.so \
                   -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python \
                   -DSERVER=ON ..
-            rerun "make -j2 >/dev/null" 3 # due to some network reasons, compilation may fail
+            rerun "make -j10 >/dev/null" 3 # due to some network reasons, compilation may fail
             check_cmd "make install -j2 >/dev/null"
             pip install -U python/dist/paddle_serving_server* >/dev/null
             ;;
@@ -118,7 +118,7 @@ function build_server() {
                   -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python \
                   -DSERVER=ON \
                   -DWITH_GPU=ON ..
-            rerun "make -j2 >/dev/null" 3 # due to some network reasons, compilation may fail
+            rerun "make -j10 >/dev/null" 3 # due to some network reasons, compilation may fail
             check_cmd "make install -j2 >/dev/null"
             pip install -U python/dist/paddle_serving_server* >/dev/null
             ;;
@@ -643,13 +643,7 @@ function python_test_yolov4(){
     cd yolov4
     case $TYPE in
         CPU)
-            python -m paddle_serving_app.package --get_model yolov4
-            tar -xzvf yolov4.tar.gz
-            check_cmd "python -m paddle_serving_server.serve --model yolov4_model/ --port 9393 &"
-            sleep 5
-            check_cmd "python test_client.py 000000570688.jpg"
-            echo "yolov4 CPU RPC inference pass"
-            kill_server_process
+            echo "no implement for cpu type"
             ;;
         GPU)
             python -m paddle_serving_app.package --get_model yolov4
@@ -670,6 +664,53 @@ function python_test_yolov4(){
     cd ..
 }
 
+function python_test_resnet50(){
+    #pwd:/ Serving/python/examples
+    local TYPE=$1
+    export SERVING_BIN=${SERVING_WORKDIR}/build-server-${TYPE}/core/general-server/serving
+    cd imagenet
+    case $TYPE in
+        CPU)
+            echo "no implement for cpu type"
+            ;;
+        GPU)
+            sh get_model.sh
+            check_cmd"python -m paddle_serving_server_gpu.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0"
+            sleep 5
+            check_cmd"python resnet50_rpc_client.py ResNet50_vd_client_config/serving_client_conf.prototxt"
+            echo "resnet50 GPU RPC inference pass"
+            kill_server_process
+            ;;
+        *)
+            echo "error type"
+            exit 1
+            ;;
+    esac
+    echo "test resnet $TYPE finished as expected"
+    unset SERVING_BIN
+    cd ..
+}
+
+function python_app_api_test(){
+    #pwd:/ Serving/python/examples
+    #test image reader
+    local TYPE=$1
+    cd imagenet
+    case $TYPE in
+        CPU)
+            check_cmd "python test_image_reader.py"
+            ;;
+        GPU)
+            echo "no implement for cpu type"
+            ;;
+        *)
+            echo "error type"
+            exit 1
+            ;;
+    esac
+    echo "test app api finised as expected"
+    cd ..
+}
 
 function python_run_test() {
     # Using the compiled binary
@@ -684,6 +725,7 @@ function python_run_test() {
     python_test_multi_fetch $TYPE # pwd: /Serving/python/examples
     python_test_yolov4 $TYPE # pwd: /Serving/python/examples
     python_test_grpc_impl $TYPE # pwd: /Serving/python/examples
+    python_test_resnet50 $TYPE # pwd: /Serving/python/examples
     echo "test python $TYPE part finished as expected."
     cd ../.. # pwd: /Serving
 }