add timeout

core/general-client/src/general_model.cpp

@@ -285,22 +285,16 @@ int PredictorClient::batch_predict(
         // int idx = _fetch_name_to_idx[name];
         if (_fetch_name_to_type[name] == 0) {
           VLOG(2) << "ferch var " << name << "type int";
-          model._int64_value_map[name].resize(
-              output.insts(0).tensor_array(idx).int64_data_size());
           int size = output.insts(0).tensor_array(idx).int64_data_size();
-          for (int i = 0; i < size; ++i) {
-            model._int64_value_map[name][i] =
-                output.insts(0).tensor_array(idx).int64_data(i);
-          }
+          model._int64_value_map[name] = std::vector<int64_t>(
+              output.insts(0).tensor_array(idx).int64_data().begin(),
+              output.insts(0).tensor_array(idx).int64_data().begin() + size);
         } else {
           VLOG(2) << "fetch var " << name << "type float";
-          model._float_value_map[name].resize(
-              output.insts(0).tensor_array(idx).float_data_size());
           int size = output.insts(0).tensor_array(idx).float_data_size();
-          for (int i = 0; i < size; ++i) {
-            model._float_value_map[name][i] =
-                output.insts(0).tensor_array(idx).float_data(i);
-          }
+          model._float_value_map[name] = std::vector<float>(
+              output.insts(0).tensor_array(idx).float_data().begin(),
+              output.insts(0).tensor_array(idx).float_data().begin() + size);
         }
         idx += 1;
       }
@@ -564,22 +558,16 @@ int PredictorClient::numpy_predict(
         // int idx = _fetch_name_to_idx[name];
         if (_fetch_name_to_type[name] == 0) {
           VLOG(2) << "ferch var " << name << "type int";
-          model._int64_value_map[name].resize(
-              output.insts(0).tensor_array(idx).int64_data_size());
           int size = output.insts(0).tensor_array(idx).int64_data_size();
-          for (int i = 0; i < size; ++i) {
-            model._int64_value_map[name][i] =
-                output.insts(0).tensor_array(idx).int64_data(i);
-          }
+          model._int64_value_map[name] = std::vector<int64_t>(
+              output.insts(0).tensor_array(idx).int64_data().begin(),
+              output.insts(0).tensor_array(idx).int64_data().begin() + size);
         } else {
           VLOG(2) << "fetch var " << name << "type float";
-          model._float_value_map[name].resize(
-              output.insts(0).tensor_array(idx).float_data_size());
           int size = output.insts(0).tensor_array(idx).float_data_size();
-          for (int i = 0; i < size; ++i) {
-            model._float_value_map[name][i] =
-                output.insts(0).tensor_array(idx).float_data(i);
-          }
+          model._float_value_map[name] = std::vector<float>(
+              output.insts(0).tensor_array(idx).float_data().begin(),
+              output.insts(0).tensor_array(idx).float_data().begin() + size);
         }
         idx += 1;
       }

doc/COMPILE.md

@@ -43,7 +43,7 @@ In the default centos7 image we provide, the Python path is `/usr/bin/python`. I
 ### Integrated CPU version paddle inference library
 
 ``` shell
-mkdir build && cd build
+mkdir server-build-cpu && cd server-build-cpu
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DSERVER=ON ..
 make -j10
 ```
@@ -53,7 +53,7 @@ you can execute `make install` to put targets under directory `./output`, you ne
 ### Integrated GPU version paddle inference library
 
 ``` shell
-mkdir build && cd build
+mkdir server-build-gpu && cd server-build-gpu
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DSERVER=ON -DWITH_GPU=ON ..
 make -j10
 ```
@@ -65,7 +65,7 @@ execute `make install` to put targets under directory `./output`
 ## Compile Client
 
 ``` shell
-mkdir build && cd build
+mkdir client-build && cd client-build
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DCLIENT=ON ..
 make -j10
 ```
@@ -75,7 +75,7 @@ execute `make install` to put targets under directory `./output`
 ## Compile the App
 
 ```bash
-mkdir build && cd build
+mkdir app-build && cd app-build
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DAPP=ON ..
 make
 ```

doc/COMPILE_CN.md

@@ -43,7 +43,7 @@ export PYTHONROOT=/usr/
 ### 集成CPU版本Paddle Inference Library
 
 ``` shell
-mkdir build && cd build
+mkdir server-build-cpu && cd server-build-cpu
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DSERVER=ON ..
 make -j10
 ```
@@ -53,7 +53,7 @@ make -j10
 ### 集成GPU版本Paddle Inference Library
 
 ``` shell
-mkdir build && cd build
+mkdir server-build-gpu && cd server-build-gpu
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DSERVER=ON -DWITH_GPU=ON ..
 make -j10
 ```
@@ -65,7 +65,7 @@ make -j10
 ## 编译Client部分
 
 ``` shell
-mkdir build && cd build
+mkdir client-build && cd client-build
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DCLIENT=ON ..
 make -j10
 ```
@@ -75,7 +75,7 @@ make -j10
 ## 编译App部分
 
 ```bash
-mkdir build && cd build
+mkdir app-build && cd app-build
 cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ -DPYTHON_LIBRARIES=$PYTHONROOT/lib/libpython2.7.so -DPYTHON_EXECUTABLE=$PYTHONROOT/bin/python -DCMAKE_INSTALL_PREFIX=./output -DAPP=ON ..
 make
 ```

doc/LATEST_PACKAGES.md

@@ -3,45 +3,45 @@
 ## CPU server
 ### Python 3
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server-0.3.0-py3-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server-0.3.1-py3-none-any.whl
 ```
 
 ### Python 2
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server-0.3.0-py2-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server-0.3.1-py2-none-any.whl
 ```
 
 ## GPU server
 ### Python 3
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server_gpu-0.3.0-py3-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server_gpu-0.3.1-py3-none-any.whl
 ```
 ### Python 2
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server_gpu-0.3.0-py2-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_server_gpu-0.3.1-py2-none-any.whl
 ```
 
 ## Client
 ### Python 3.7
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.0-cp37-none-manylinux1_x86_64.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.1-cp37-none-any.whl
 ```
 ### Python 3.6
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.0-cp36-none-manylinux1_x86_64.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.1-cp36-none-any.whl
 ```
 ### Python 2.7
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.0-cp27-none-manylinux1_x86_64.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.3.1-cp27-none-any.whl
 ```
 
 ## App
 ### Python 3
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_app-0.1.0-py3-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_app-0.1.1-py3-none-any.whl
 ```
 
 ### Python 2
 ```
-https://paddle-serving.bj.bcebos.com/whl/paddle_serving_app-0.1.0-py2-none-any.whl
+https://paddle-serving.bj.bcebos.com/whl/paddle_serving_app-0.1.1-py2-none-any.whl
 ```

python/paddle_serving_app/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving App version string """
-serving_app_version = "0.1.0"
+serving_app_version = "0.1.1"

python/paddle_serving_client/__init__.py

@@ -384,6 +384,7 @@ class Client(object):
 class MultiLangClient(object):
     def __init__(self):
         self.channel_ = None
+        self.rpc_timeout_ms_ = 2000
 
     def load_client_config(self, path):
         if not isinstance(path, str):
@@ -395,13 +396,18 @@ class MultiLangClient(object):
         pass
 
     def set_rpc_timeout_ms(self, rpc_timeout):
-        # TODO
-        pass
+        self.rpc_timeout_ms_ = rpc_timeout
 
     def connect(self, endpoints):
-        g_endpoint = [(endpoint.spilt(':')[0], endpoint.split(':')[1])
-                      for endpoint in endpoints]
-        self.channel_ = grpc.insecure_channel(g_endpoint)
+        # https://github.com/tensorflow/serving/issues/1382
+        options = [('grpc.max_receive_message_length', 512 * 1024 * 1024),
+                   ('grpc.max_send_message_length', 512 * 1024 * 1024)]
+
+        # TODO
+        if len(endpoints) > 1:
+            print("Warn: grpc can only set a endpoint")
+        g_endpoint = endpoints[0]
+        self.channel_ = grpc.insecure_channel(g_endpoint, options=options)
         self.stub_ = multi_lang_general_model_service_pb2_grpc.MultiLangGeneralModelServiceStub(
             self.channel_)
 
@@ -534,17 +540,20 @@ class MultiLangClient(object):
                 fetch,
                 need_variant_tag=False,
                 asyn=False,
-                is_python=True):
+                is_python=True,
+                timeout=None):
+        if timeout is None:
+            timeout = self.rpc_timeout_ms_
         req = self._pack_feed_data(feed, fetch, is_python=is_python)
         if not asyn:
-            resp = self.stub_.inference(req)
+            resp = self.stub_.inference(req, timeout=timeout)
             return self._unpack_resp(
                 resp,
                 fetch,
                 is_python=is_python,
                 need_variant_tag=need_variant_tag)
         else:
-            call_future = self.stub_.inference.future(req)
+            call_future = self.stub_.inference.future(req, timeout=timeout)
             return MultiLangPredictFuture(
                 call_future,
                 self._done_callback_func(

python/paddle_serving_client/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.0"
-serving_server_version = "0.3.0"
-module_proto_version = "0.3.0"
+serving_client_version = "0.3.1"
+serving_server_version = "0.3.1"
+module_proto_version = "0.3.1"

python/paddle_serving_server/__init__.py

@@ -440,12 +440,13 @@ class Server(object):
         os.system(command)
 
 
-class MultiLangServerService(
-        multi_lang_general_model_service_pb2_grpc.MultiLangGeneralModelService):
-    def __init__(self, model_config_path, endpoints):
+class MultiLangServerService(multi_lang_general_model_service_pb2_grpc.
+                             MultiLangGeneralModelServiceServier):
+    def __init__(self, model_config_path, endpoints, rpc_timeout_ms):
         from paddle_serving_client import Client
         self._parse_model_config(model_config_path)
         self.bclient_ = Client()
+        self.bclient_.set_rpc_timeout_ms(rpc_timeout_ms)
         self.bclient_.load_client_config(
             "{}/serving_server_conf.prototxt".format(model_config_path))
         self.bclient_.connect(endpoints)
@@ -544,18 +545,26 @@ class MultiLangServerService(
 
 
 class MultiLangServer(object):
-    def __init__(self, worker_num=2):
+    def __init__(self):
         self.bserver_ = Server()
-        self.worker_num_ = worker_num
+        self.worker_num_ = 4
+        self.body_size_ = 512 * 1024 * 1024
+        self.concurrency_ = 100
+        self.rpc_timeout_ms_ = 20000
+
+    def set_rpc_timeout_ms(self, rpc_timeout_ms):
+        self.rpc_timeout_ms_ = rpc_timeout_ms
 
     def set_max_concurrency(self, concurrency):
+        self.concurrency_ = concurrency
         self.bserver_.set_max_concurrency(concurrency)
 
     def set_num_threads(self, threads):
+        self.worker_num_ = threads
         self.bserver_.set_num_threads(threads)
 
     def set_max_body_size(self, body_size):
-        # TODO: grpc body
+        self.body_size_ = body_size
         self.bserver_.set_max_body_size(body_size)
 
     def set_port(self, port):
@@ -619,12 +628,16 @@ class MultiLangServer(object):
         p_bserver = Process(
             target=self._launch_brpc_service, args=(self.bserver_, ))
         p_bserver.start()
+        options = [('grpc.max_send_message_length', self.body_size_),
+                   ('grpc.max_receive_message_length', self.body_size_)]
         server = grpc.server(
-            futures.ThreadPoolExecutor(max_workers=self.worker_num_))
+            futures.ThreadPoolExecutor(max_workers=self.worker_num_),
+            options=options,
+            maximum_concurrent_rpcs=self.concurrency_)
         multi_lang_general_model_service_pb2_grpc.add_MultiLangGeneralModelServiceServicer_to_server(
             MultiLangServerService(self.model_config_path_,
-                                   ["0.0.0.0:{}".format(self.port_list_[0])]),
-            server)
+                                   ["0.0.0.0:{}".format(self.port_list_[0])],
+                                   self.rpc_timeout_ms_), server)
         server.add_insecure_port('[::]:{}'.format(self.gport_))
         server.start()
         p_bserver.join()

python/paddle_serving_server/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.0"
-serving_server_version = "0.3.0"
-module_proto_version = "0.3.0"
+serving_client_version = "0.3.1"
+serving_server_version = "0.3.1"
+module_proto_version = "0.3.1"

python/paddle_serving_server_gpu/version.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Paddle Serving Client version string """
-serving_client_version = "0.3.0"
-serving_server_version = "0.3.0"
-module_proto_version = "0.3.0"
+serving_client_version = "0.3.1"
+serving_server_version = "0.3.1"
+module_proto_version = "0.3.1"

tools/python_tag.py

@@ -15,6 +15,6 @@
 from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag
 import re
 with open("setup.cfg", "w") as f:
-    line = "[bdist_wheel]\npython-tag={0}{1}\nplat-name=manylinux1_x86_64".format(
-        get_abbr_impl(), get_impl_ver())
+    line = "[bdist_wheel]\npython-tag={0}{1}".format(get_abbr_impl(),
+                                                     get_impl_ver())
     f.write(line)