Merge pull request #3 from PaddlePaddle/develop

Sync with Origin Repo

README.md

@@ -154,10 +154,87 @@ curl -H "Content-Type:application/json" -X POST -d '{"url": "https://paddle-serv
 {"label":"daisy","prob":0.9341403245925903}
 ```
 
+<h3 align="center">More Demos</h4>
 
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | Bert-Base-Baike                                              |
+| URL                | [https://paddle-serving.bj.bcebos.com/bert_example/bert_seq128.tar.gz](https://paddle-serving.bj.bcebos.com/bert_example%2Fbert_seq128.tar.gz) |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/bert |
+| Description        | Get semantic representation from a Chinese Sentence          |
 
 
 
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | Resnet50-Imagenet                                            |
+| URL                | [https://paddle-serving.bj.bcebos.com/imagenet-example/ResNet50_vd.tar.gz](https://paddle-serving.bj.bcebos.com/imagenet-example%2FResNet50_vd.tar.gz) |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/imagenet |
+| Description        | Get image semantic representation from an image              |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | Resnet101-Imagenet                                           |
+| URL                | https://paddle-serving.bj.bcebos.com/imagenet-example/ResNet101_vd.tar.gz |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/imagenet |
+| Description        | Get image semantic representation from an image              |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | CNN-IMDB                                                     |
+| URL                | https://paddle-serving.bj.bcebos.com/imdb-demo/imdb_model.tar.gz |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/imdb |
+| Description        | Get category probability from an English Sentence            |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | LSTM-IMDB                                                    |
+| URL                | https://paddle-serving.bj.bcebos.com/imdb-demo/imdb_model.tar.gz |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/imdb |
+| Description        | Get category probability from an English Sentence            |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | BOW-IMDB                                                     |
+| URL                | https://paddle-serving.bj.bcebos.com/imdb-demo/imdb_model.tar.gz |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/imdb |
+| Description        | Get category probability from an English Sentence            |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | Jieba-LAC                                                    |
+| URL                | https://paddle-serving.bj.bcebos.com/lac/lac_model.tar.gz    |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/lac |
+| Description        | Get word segmentation from a Chinese Sentence                |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | DNN-CTR                                                      |
+| URL                | None(Get model by [local_train.py](./python/examples/criteo_ctr/local_train.py))                            |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/criteo_ctr |
+| Description        | Get click probability from a feature vector of item          |
+
+
+
+| Key                | Value                                                        |
+| :----------------- | :----------------------------------------------------------- |
+| Model Name         | DNN-CTR(with cube)                                           |
+| URL                | None(Get model by [local_train.py](python/examples/criteo_ctr_with_cube/local_train.py))                            |
+| Client/Server Code | https://github.com/PaddlePaddle/Serving/tree/develop/python/examples/criteo_ctr_with_cube |
+| Description        | Get click probability from a feature vector of item          |
+
 <h2 align="center">Document</h2>
 
 ### New to Paddle Serving

doc/RUN_IN_DOCKER.md

@@ -147,7 +147,7 @@ tar -xzf uci_housing.tar.gz
   Running on the Server side (inside the container):
 
   ```bash
-  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --name uci
+  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --name uci --gpu_ids 0
   ```
 
   Running on the Client side (inside or outside the container):
@@ -161,7 +161,7 @@ tar -xzf uci_housing.tar.gz
   Running on the Server side (inside the container):
 
   ```bash
-  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292
+  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --gpu_ids 0
   ```
 
   Running following Python code on the Client side (inside or outside the container, The `paddle-serving-client` package needs to be installed):

doc/RUN_IN_DOCKER_CN.md

@@ -145,7 +145,7 @@ tar -xzf uci_housing.tar.gz
   在Server端（容器内）运行：
 
   ```bash
-  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --name uci 
+  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --name uci --gpu_ids 0
   ```
 
   在Client端（容器内或容器外）运行：
@@ -159,7 +159,7 @@ tar -xzf uci_housing.tar.gz
   在Server端（容器内）运行：
 
   ```bash
-  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292
+  python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9292 --gpu_ids 0
   ```
 
   在Client端（容器内或容器外，需要安装`paddle-serving-client`包）运行下面Python代码：

python/examples/criteo_ctr/test_client.py

@@ -21,6 +21,10 @@ import time
 import criteo_reader as criteo
 from paddle_serving_client.metric import auc
 
+import sys
+
+py_version = sys.version_info[0]
+
 client = Client()
 client.load_client_config(sys.argv[1])
 client.connect(["127.0.0.1:9292"])
@@ -39,7 +43,10 @@ label_list = []
 prob_list = []
 start = time.time()
 for ei in range(1000):
+    if py_version == 2:
         data = reader().next()
+    else:
+        data = reader().__next__()
     feed_dict = {}
     for i in range(1, 27):
         feed_dict["sparse_{}".format(i - 1)] = data[0][i]

python/examples/criteo_ctr_with_cube/test_server_gpu.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=doc-string-missing
+
+import os
+import sys
+from paddle_serving_server_gpu import OpMaker
+from paddle_serving_server_gpu import OpSeqMaker
+from paddle_serving_server_gpu import Server
+
+op_maker = OpMaker()
+read_op = op_maker.create('general_reader')
+general_dist_kv_infer_op = op_maker.create('general_dist_kv_infer')
+response_op = op_maker.create('general_response')
+
+op_seq_maker = OpSeqMaker()
+op_seq_maker.add_op(read_op)
+op_seq_maker.add_op(general_dist_kv_infer_op)
+op_seq_maker.add_op(response_op)
+
+server = Server()
+server.set_op_sequence(op_seq_maker.get_op_sequence())
+server.set_num_threads(4)
+server.load_model_config(sys.argv[1])
+server.prepare_server(workdir="work_dir1", port=9292, device="cpu")
+server.run_server()

python/examples/imagenet/image_http_client.py

@@ -17,10 +17,16 @@ import base64
 import json
 import time
 import os
+import sys
+
+py_version = sys.version_info[0]
 
 
 def predict(image_path, server):
+    if py_version == 2:
         image = base64.b64encode(open(image_path).read())
+    else:
+        image = base64.b64encode(open(image_path, "rb").read()).decode("utf-8")
     req = json.dumps({"image": image, "fetch": ["score"]})
     r = requests.post(
         server, data=req, headers={"Content-Type": "application/json"})
@@ -28,18 +34,8 @@ def predict(image_path, server):
     return r
 
 
-def batch_predict(image_path, server):
-    image = base64.b64encode(open(image_path).read())
-    req = json.dumps({"image": [image, image], "fetch": ["score"]})
-    r = requests.post(
-        server, data=req, headers={"Content-Type": "application/json"})
-    print(r.json()["result"][1]["score"][0])
-    return r
-
-
 if __name__ == "__main__":
     server = "http://127.0.0.1:9393/image/prediction"
-    #image_path = "./data/n01440764_10026.JPEG"
     image_list = os.listdir("./image_data/n01440764/")
     start = time.time()
     for img in image_list:

python/examples/imagenet/image_rpc_client.py

@@ -19,16 +19,15 @@ import time
 
 client = Client()
 client.load_client_config(sys.argv[1])
-client.connect(["127.0.0.1:9295"])
+client.connect(["127.0.0.1:9393"])
 reader = ImageReader()
 
 start = time.time()
 for i in range(1000):
-    with open("./data/n01440764_10026.JPEG") as f:
+    with open("./data/n01440764_10026.JPEG", "rb") as f:
         img = f.read()
     img = reader.process_image(img).reshape(-1)
     fetch_map = client.predict(feed={"image": img}, fetch=["score"])
-    print(i)
 end = time.time()
 print(end - start)
 

python/examples/imdb/imdb_reader.py

@@ -19,15 +19,23 @@ import paddle
 import re
 import paddle.fluid.incubate.data_generator as dg
 
+py_version = sys.version_info[0]
+
 
 class IMDBDataset(dg.MultiSlotDataGenerator):
     def load_resource(self, dictfile):
         self._vocab = {}
         wid = 0
+        if py_version == 2:
             with open(dictfile) as f:
                 for line in f:
                     self._vocab[line.strip()] = wid
                     wid += 1
+        else:
+            with open(dictfile, encoding="utf-8") as f:
+                for line in f:
+                    self._vocab[line.strip()] = wid
+                    wid += 1
         self._unk_id = len(self._vocab)
         self._pattern = re.compile(r'(;|,|\.|\?|!|\s|\(|\))')
         self.return_value = ("words", [1, 2, 3, 4, 5, 6]), ("label", [0])

python/paddle_serving_client/metric/__init__.py

@@ -11,5 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from auc import auc
-from acc import acc
+from .auc import auc
+from .acc import acc

python/paddle_serving_server_gpu/__init__.py

@@ -55,6 +55,7 @@ class OpMaker(object):
             "general_text_reader": "GeneralTextReaderOp",
             "general_text_response": "GeneralTextResponseOp",
             "general_single_kv": "GeneralSingleKVOp",
+            "general_dist_kv_infer": "GeneralDistKVInferOp",
             "general_dist_kv": "GeneralDistKVOp"
         }
 
@@ -104,6 +105,7 @@ class Server(object):
         self.infer_service_fn = "infer_service.prototxt"
         self.model_toolkit_fn = "model_toolkit.prototxt"
         self.general_model_config_fn = "general_model.prototxt"
+        self.cube_config_fn = "cube.conf"
         self.workdir = ""
         self.max_concurrency = 0
         self.num_threads = 4
@@ -184,6 +186,11 @@ class Server(object):
                       "w") as fout:
                 fout.write(str(self.model_conf))
             self.resource_conf = server_sdk.ResourceConf()
+            for workflow in self.workflow_conf.workflows:
+                for node in workflow.nodes:
+                    if "dist_kv" in node.name:
+                        self.resource_conf.cube_config_path = workdir
+                        self.resource_conf.cube_config_file = self.cube_config_fn
             self.resource_conf.model_toolkit_path = workdir
             self.resource_conf.model_toolkit_file = self.model_toolkit_fn
             self.resource_conf.general_model_path = workdir

python/paddle_serving_server_gpu/serve.py

@@ -71,7 +71,7 @@ def start_multi_card(args):  # pylint: disable=doc-string-missing
     else:
         gpus = args.gpu_ids.split(",")
     if len(gpus) <= 0:
-        start_gpu_card_model(-1, args)
+        start_gpu_card_model(-1, 0, args)
     else:
         gpu_processes = []
         for i, gpu_id in enumerate(gpus):

python/setup.py.client.in

@@ -18,6 +18,7 @@ from __future__ import print_function
 
 import platform
 import os
+import sys
 
 from setuptools import setup, Distribution, Extension
 from setuptools import find_packages
@@ -25,6 +26,7 @@ from setuptools import setup
 from paddle_serving_client.version import serving_client_version
 from pkg_resources import DistributionNotFound, get_distribution
 
+py_version = sys.version_info[0]
         
 def python_version():
     return [int(v) for v in platform.python_version().split(".")]
@@ -37,8 +39,9 @@ def find_package(pkgname):
         return False
 
 def copy_lib():
+    lib_list = ['libpython2.7.so.1.0', 'libssl.so.10', 'libcrypto.so.10'] if py_version == 2 else ['libpython3.6m.so.1.0', 'libssl.so.10', 'libcrypto.so.10']
     os.popen('mkdir -p paddle_serving_client/lib')
-    for lib in ['libpython2.7.so.1.0', 'libssl.so.10', 'libcrypto.so.10']:
+    for lib in lib_list:
         r = os.popen('whereis {}'.format(lib))
         text = r.read()
         os.popen('cp {} ./paddle_serving_client/lib'.format(text.strip().split(' ')[1]))

tools/serving_build.sh

@@ -211,12 +211,11 @@ function python_run_criteo_ctr_with_cube() {
             cp ../../../build-server-$TYPE/output/bin/cube* ./cube/ 
             mkdir -p $PYTHONROOT/lib/python2.7/site-packages/paddle_serving_server/serving-cpu-avx-openblas-0.1.3/
             yes | cp ../../../build-server-$TYPE/output/demo/serving/bin/serving $PYTHONROOT/lib/python2.7/site-packages/paddle_serving_server/serving-cpu-avx-openblas-0.1.3/
-
             sh cube_prepare.sh &
             check_cmd "mkdir work_dir1 && cp cube/conf/cube.conf ./work_dir1/"    
             python test_server.py ctr_serving_model_kv &
             check_cmd "python test_client.py ctr_client_conf/serving_client_conf.prototxt ./ut_data >score"
-            tail -n 2 score
+            tail -n 2 score | awk 'NR==1'
             AUC=$(tail -n 2  score | awk 'NR==1')
             VAR2="0.67" #TODO: temporarily relax the threshold to 0.67
             RES=$( echo "$AUC>$VAR2" | bc )
@@ -229,6 +228,30 @@ function python_run_criteo_ctr_with_cube() {
             ps -ef | grep "cube" | grep -v grep | awk '{print $2}' | xargs kill
             ;;
         GPU)
+            check_cmd "wget https://paddle-serving.bj.bcebos.com/unittest/ctr_cube_unittest.tar.gz"
+            check_cmd "tar xf ctr_cube_unittest.tar.gz"
+            check_cmd "mv models/ctr_client_conf ./"
+            check_cmd "mv models/ctr_serving_model_kv ./"
+            check_cmd "mv models/data ./cube/"
+            check_cmd "mv models/ut_data ./"
+            cp ../../../build-server-$TYPE/output/bin/cube* ./cube/
+            mkdir -p $PYTHONROOT/lib/python2.7/site-packages/paddle_serving_server_gpu/serving-gpu-0.1.3/
+            yes | cp ../../../build-server-$TYPE/output/demo/serving/bin/serving $PYTHONROOT/lib/python2.7/site-packages/paddle_serving_server_gpu/serving-gpu-0.1.3/
+            sh cube_prepare.sh &
+            check_cmd "mkdir work_dir1 && cp cube/conf/cube.conf ./work_dir1/"
+            python test_server_gpu.py ctr_serving_model_kv &
+            check_cmd "python test_client.py ctr_client_conf/serving_client_conf.prototxt ./ut_data >score"
+            tail -n 2 score | awk 'NR==1'
+            AUC=$(tail -n 2  score | awk 'NR==1')
+            VAR2="0.67" #TODO: temporarily relax the threshold to 0.67
+            RES=$( echo "$AUC>$VAR2" | bc )
+            if [[ $RES -eq 0 ]]; then
+                echo "error with criteo_ctr_with_cube inference auc test, auc should > 0.70"
+                exit 1
+            fi
+            echo "criteo_ctr_with_cube inference auc test success"
+            ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill
+            ps -ef | grep "cube" | grep -v grep | awk '{print $2}' | xargs kill
             ;;
         *)
             echo "error type"