From cfddcf6fb401ad20a1ee51777ed5675aa55de1db Mon Sep 17 00:00:00 2001 From: Jiawei Wang Date: Wed, 3 Feb 2021 15:41:13 +0800 Subject: [PATCH] Merge pull request #1014 from HexToString/fix_grpc_bug fix grpc_impl_bug and add readme by HexToString --- .../examples/grpc_impl_example/imdb/README.md | 25 +++++++++ .../grpc_impl_example/imdb/README_CN.md | 24 ++++++++ .../imdb/test_client.py} | 28 ++++++---- .../imdb/test_multilang_ensemble_client.py | 39 ------------- .../imdb/test_multilang_ensemble_server.py | 40 ------------- python/examples/imdb/test_ensemble_server.py | 40 ------------- python/paddle_serving_server/__init__.py | 56 ++++++++++++------- python/paddle_serving_server_gpu/__init__.py | 13 ++++- 8 files changed, 111 insertions(+), 154 deletions(-) create mode 100644 python/examples/grpc_impl_example/imdb/README.md create mode 100644 python/examples/grpc_impl_example/imdb/README_CN.md rename python/examples/{imdb/test_ensemble_client.py => grpc_impl_example/imdb/test_client.py} (63%) delete mode 100644 python/examples/grpc_impl_example/imdb/test_multilang_ensemble_client.py delete mode 100644 python/examples/grpc_impl_example/imdb/test_multilang_ensemble_server.py delete mode 100644 python/examples/imdb/test_ensemble_server.py diff --git a/python/examples/grpc_impl_example/imdb/README.md b/python/examples/grpc_impl_example/imdb/README.md new file mode 100644 index 00000000..73636f3c --- /dev/null +++ b/python/examples/grpc_impl_example/imdb/README.md @@ -0,0 +1,25 @@ +## IMDB comment sentiment inference service + +([简体中文](./README_CN.md)|English) + +### Get model files and sample data + +``` +sh get_data.sh +``` +the package downloaded contains cnn, lstm and bow model config along with their test_data and train_data. + +### Start RPC inference service + +``` +python -m paddle_serving_server.serve --model imdb_cnn_model/ --thread 10 --port 9393 --use_multilang +``` +### RPC Infer + +The `paddlepaddle` package is used in `test_client.py`, and you may need to download the corresponding package(`pip install paddlepaddle`). + +``` +head test_data/part-0 | python test_client.py +``` + +it will get predict results of the first 10 test cases. diff --git a/python/examples/grpc_impl_example/imdb/README_CN.md b/python/examples/grpc_impl_example/imdb/README_CN.md new file mode 100644 index 00000000..327b1c55 --- /dev/null +++ b/python/examples/grpc_impl_example/imdb/README_CN.md @@ -0,0 +1,24 @@ +## IMDB评论情绪预测服务 + +(简体中文|[English](./README.md)) + +### 获取模型文件和样例数据 + +``` +sh get_data.sh +``` +脚本会下载和解压出cnn、lstm和bow三种模型的配置文文件以及test_data和train_data。 + +### 启动RPC预测服务 + +``` +python -m paddle_serving_server.serve --model imdb_cnn_model/ --thread 10 --port 9393 --use_multilang +``` +### 执行预测 + +`test_client.py`中使用了`paddlepaddle`包，需要进行下载（`pip install paddlepaddle`）。 + +``` +head test_data/part-0 | python test_client.py +``` +预测test_data/part-0的前十个样例。 diff --git a/python/examples/imdb/test_ensemble_client.py b/python/examples/grpc_impl_example/imdb/test_client.py similarity index 63% rename from python/examples/imdb/test_ensemble_client.py rename to python/examples/grpc_impl_example/imdb/test_client.py index eb1e29dd..bddc4d50 100644 --- a/python/examples/imdb/test_ensemble_client.py +++ b/python/examples/grpc_impl_example/imdb/test_client.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=doc-string-missing - -from paddle_serving_client import Client -from imdb_reader import IMDBDataset +from paddle_serving_client import MultiLangClient as Client +from paddle_serving_app.reader.imdb_reader import IMDBDataset +import sys +import numpy as np client = Client() -# If you have more than one model, make sure that the input -# and output of more than one model are the same. -client.load_client_config('imdb_bow_client_conf/serving_client_conf.prototxt') client.connect(["127.0.0.1:9393"]) # you can define any english sentence or dataset here @@ -28,11 +26,17 @@ client.connect(["127.0.0.1:9393"]) imdb_dataset = IMDBDataset() imdb_dataset.load_resource('imdb.vocab') -for i in range(3): - line = 'i am very sad | 0' +for line in sys.stdin: word_ids, label = imdb_dataset.get_words_and_label(line) - feed = {"words": word_ids} + word_len = len(word_ids) + feed = { + "words": np.array(word_ids).reshape(word_len, 1), + "words.lod": [0, word_len] + } fetch = ["prediction"] - fetch_maps = client.predict(feed=feed, fetch=fetch) - for model, fetch_map in fetch_maps.items(): - print("step: {}, model: {}, res: {}".format(i, model, fetch_map)) + fetch_map = client.predict(feed=feed, fetch=fetch, batch=True) + if fetch_map["serving_status_code"] == 0: + print(fetch_map) + else: + print(fetch_map["serving_status_code"]) + #print("{} {}".format(fetch_map["prediction"][0], label[0])) diff --git a/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_client.py b/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_client.py deleted file mode 100644 index 43034e49..00000000 --- a/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_client.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=doc-string-missing - -from paddle_serving_client import MultiLangClient -from imdb_reader import IMDBDataset - -client = MultiLangClient() -# If you have more than one model, make sure that the input -# and output of more than one model are the same. -client.connect(["127.0.0.1:9393"]) - -# you can define any english sentence or dataset here -# This example reuses imdb reader in training, you -# can define your own data preprocessing easily. -imdb_dataset = IMDBDataset() -imdb_dataset.load_resource('imdb.vocab') - -for i in range(3): - line = 'i am very sad | 0' - word_ids, label = imdb_dataset.get_words_and_label(line) - feed = {"words": word_ids} - fetch = ["prediction"] - fetch_maps = client.predict(feed=feed, fetch=fetch) - for model, fetch_map in fetch_maps.items(): - if model == "serving_status_code": - continue - print("step: {}, model: {}, res: {}".format(i, model, fetch_map)) diff --git a/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_server.py b/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_server.py deleted file mode 100644 index 053aa06f..00000000 --- a/python/examples/grpc_impl_example/imdb/test_multilang_ensemble_server.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=doc-string-missing - -from paddle_serving_server import OpMaker -from paddle_serving_server import OpGraphMaker -from paddle_serving_server import MultiLangServer - -op_maker = OpMaker() -read_op = op_maker.create('general_reader') -cnn_infer_op = op_maker.create( - 'general_infer', engine_name='cnn', inputs=[read_op]) -bow_infer_op = op_maker.create( - 'general_infer', engine_name='bow', inputs=[read_op]) -response_op = op_maker.create( - 'general_response', inputs=[cnn_infer_op, bow_infer_op]) - -op_graph_maker = OpGraphMaker() -op_graph_maker.add_op(read_op) -op_graph_maker.add_op(cnn_infer_op) -op_graph_maker.add_op(bow_infer_op) -op_graph_maker.add_op(response_op) - -server = MultiLangServer() -server.set_op_graph(op_graph_maker.get_op_graph()) -model_config = {cnn_infer_op: 'imdb_cnn_model', bow_infer_op: 'imdb_bow_model'} -server.load_model_config(model_config) -server.prepare_server(workdir="work_dir1", port=9393, device="cpu") -server.run_server() diff --git a/python/examples/imdb/test_ensemble_server.py b/python/examples/imdb/test_ensemble_server.py deleted file mode 100644 index 464288a0..00000000 --- a/python/examples/imdb/test_ensemble_server.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=doc-string-missing - -from paddle_serving_server import OpMaker -from paddle_serving_server import OpGraphMaker -from paddle_serving_server import Server - -op_maker = OpMaker() -read_op = op_maker.create('general_reader') -cnn_infer_op = op_maker.create( - 'general_infer', engine_name='cnn', inputs=[read_op]) -bow_infer_op = op_maker.create( - 'general_infer', engine_name='bow', inputs=[read_op]) -response_op = op_maker.create( - 'general_response', inputs=[cnn_infer_op, bow_infer_op]) - -op_graph_maker = OpGraphMaker() -op_graph_maker.add_op(read_op) -op_graph_maker.add_op(cnn_infer_op) -op_graph_maker.add_op(bow_infer_op) -op_graph_maker.add_op(response_op) - -server = Server() -server.set_op_graph(op_graph_maker.get_op_graph()) -model_config = {cnn_infer_op: 'imdb_cnn_model', bow_infer_op: 'imdb_bow_model'} -server.load_model_config(model_config) -server.prepare_server(workdir="work_dir1", port=9393, device="cpu") -server.run_server() diff --git a/python/paddle_serving_server/__init__.py b/python/paddle_serving_server/__init__.py index 5ef3cf75..4d0832b6 100644 --- a/python/paddle_serving_server/__init__.py +++ b/python/paddle_serving_server/__init__.py @@ -537,26 +537,37 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. fetch_names = list(request.fetch_var_names) is_python = request.is_python log_id = request.log_id - feed_dict = {} - feed_inst = request.insts[0] - for idx, name in enumerate(feed_names): - var = feed_inst.tensor_array[idx] - v_type = self.feed_types_[name] - data = None - if is_python: - if v_type == 0: # int64 - data = np.frombuffer(var.data, dtype="int64") - elif v_type == 1: # float32 - data = np.frombuffer(var.data, dtype="float32") - elif v_type == 2: # int32 - data = np.frombuffer(var.data, dtype="int32") + feed_batch = [] + for feed_inst in request.insts: + feed_dict = {} + for idx, name in enumerate(feed_names): + var = feed_inst.tensor_array[idx] + v_type = self.feed_types_[name] + data = None + if is_python: + if v_type == 0: # int64 + data = np.frombuffer(var.data, dtype="int64") + elif v_type == 1: # float32 + data = np.frombuffer(var.data, dtype="float32") + elif v_type == 2: # int32 + data = np.frombuffer(var.data, dtype="int32") + else: + raise Exception("error type.") else: - raise Exception("error type.") - data.shape = list(feed_inst.tensor_array[idx].shape) - feed_dict[name] = data - if len(var.lod) > 0: - feed_dict["{}.lod".format()] = var.lod - return feed_dict, fetch_names, is_python, log_id + if v_type == 0: # int64 + data = np.array(list(var.int64_data), dtype="int64") + elif v_type == 1: # float32 + data = np.array(list(var.float_data), dtype="float32") + elif v_type == 2: # int32 + data = np.array(list(var.int_data), dtype="int32") + else: + raise Exception("error type.") + data.shape = list(feed_inst.tensor_array[idx].shape) + feed_dict[name] = data + if len(var.lod) > 0: + feed_dict["{}.lod".format(name)] = var.lod + feed_batch.append(feed_dict) + return feed_batch, fetch_names, is_python, log_id def _pack_inference_response(self, ret, fetch_names, is_python): resp = multi_lang_general_model_service_pb2.InferenceResponse() @@ -608,10 +619,10 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. return resp def Inference(self, request, context): - feed_dict, fetch_names, is_python, log_id = \ + feed_batch, fetch_names, is_python, log_id = \ self._unpack_inference_request(request) ret = self.bclient_.predict( - feed=feed_dict, + feed=feed_batch, fetch=fetch_names, batch=True, need_variant_tag=True, @@ -649,6 +660,9 @@ class MultiLangServer(object): "max_body_size is less than default value, will use default value in service." ) + def use_encryption_model(self, flag=False): + self.encryption_model = flag + def set_port(self, port): self.gport_ = port diff --git a/python/paddle_serving_server_gpu/__init__.py b/python/paddle_serving_server_gpu/__init__.py index ffbb888e..9f71138e 100644 --- a/python/paddle_serving_server_gpu/__init__.py +++ b/python/paddle_serving_server_gpu/__init__.py @@ -244,6 +244,9 @@ class Server(object): "max_body_size is less than default value, will use default value in service." ) + def use_encryption_model(self, flag=False): + self.encryption_model = flag + def set_port(self, port): self.port = port @@ -690,6 +693,8 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. raise Exception("error type.") data.shape = list(feed_inst.tensor_array[idx].shape) feed_dict[name] = data + if len(var.lod) > 0: + feed_dict["{}.lod".format(name)] = var.lod feed_batch.append(feed_dict) return feed_batch, fetch_names, is_python, log_id @@ -744,11 +749,12 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. return resp def Inference(self, request, context): - feed_dict, fetch_names, is_python, log_id \ + feed_batch, fetch_names, is_python, log_id \ = self._unpack_inference_request(request) ret = self.bclient_.predict( - feed=feed_dict, + feed=feed_batch, fetch=fetch_names, + batch=True, need_variant_tag=True, log_id=log_id) return self._pack_inference_response(ret, fetch_names, is_python) @@ -787,6 +793,9 @@ class MultiLangServer(object): "max_body_size is less than default value, will use default value in service." ) + def use_encryption_model(self, flag=False): + self.encryption_model = flag + def set_port(self, port): self.gport_ = port -- GitLab