diff --git a/README.md b/README.md index efa6765dcc295ed27bb44d6d014a951f31d8cc84..870f2f587f35ba1526c49f58698ae4db17ff0f81 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ python -m paddle_serving_server.serve --model uci_housing_model --thread 10 --po | `port` | int | `9292` | Exposed port of current service to users| | `name` | str | `""` | Service name, can be used to generate HTTP request url | | `model` | str | `""` | Path of paddle model directory to be served | -| `mem_optim` | - | - | Enable memory / graphic memory optimization | +| `mem_optim_off` | - | - | Disable memory / graphic memory optimization | | `ir_optim` | - | - | Enable analysis and optimization of calculation graph | | `use_mkl` (Only for cpu version) | - | - | Run inference with MKL | diff --git a/README_CN.md b/README_CN.md index 6ab74ad358e42134bde0aff25548900f6af90c73..6317a79513a3d5e3247d249885d8bfe06de0e1c9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -120,7 +120,7 @@ python -m paddle_serving_server.serve --model uci_housing_model --thread 10 --po | `port` | int | `9292` | Exposed port of current service to users| | `name` | str | `""` | Service name, can be used to generate HTTP request url | | `model` | str | `""` | Path of paddle model directory to be served | -| `mem_optim` | - | - | Enable memory optimization | +| `mem_optim_off` | - | - | Disable memory optimization | | `ir_optim` | - | - | Enable analysis and optimization of calculation graph | | `use_mkl` (Only for cpu version) | - | - | Run inference with MKL | diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake index f5ef70379a5562617e77a9e2ff46587cd48a0f6c..39412f6950b7d4fe71f294079b69707b202f0876 100644 --- a/cmake/external/brpc.cmake +++ b/cmake/external/brpc.cmake @@ -40,8 +40,8 @@ ExternalProject_Add( extern_brpc ${EXTERNAL_PROJECT_LOG_ARGS} # TODO(gongwb): change to de newst repo when they changed. - GIT_REPOSITORY "https://github.com/gongweibao/brpc" - GIT_TAG "e9b67ec1b7458f2af5fae76451afe1e27e01b4b4" + GIT_REPOSITORY "https://github.com/wangjiawei04/brpc" + GIT_TAG "6d79e0b17f25107c35b705ea58d888083f59ff47" PREFIX ${BRPC_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/doc/FAQ.md b/doc/FAQ.md index 3bdd2dfd4739b54bf39b6b3f561c43bab3edabde..eb4f05a28594effcf59aac880cf4d81846a3a925 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -12,4 +12,7 @@ client.load_client_config(sys.argv[1]) client.set_rpc_timeout_ms(100000) client.connect(["127.0.0.1:9393"]) - ``` + ``` + +- Q: 如何使用自己编译的Paddle Serving进行预测？ + A：通过pip命令安装自己编译出的whl包，并设置SERVING_BIN环境变量为编译出的serving二进制文件路径。 diff --git a/doc/PERFORMANCE_OPTIM.md b/doc/PERFORMANCE_OPTIM.md index 651be1c139b5960fa287fc3e981f3039f9f098a2..e87e9541cccadf318821807aa63ca4b0e6809a1b 100644 --- a/doc/PERFORMANCE_OPTIM.md +++ b/doc/PERFORMANCE_OPTIM.md @@ -14,7 +14,35 @@ Under the same conditions, the communication time of the HTTP prediction service Parameters for performance optimization: +The memory/graphic memory optimization option is enabled by default in Paddle Serving, which can reduce the memory/video memory usage and usually does not affect performance. If you need to turn it off, you can use --mem_optim_off in the command line. + +r_optim can optimize the calculation graph and increase the inference speed. It is turned off by default and turned on by --ir_optim in the command line. + | Parameters | Type | Default | Description | | ---------- | ---- | ------- | ------------------------------------------------------------ | -| mem_optim | - | - | Enable memory / graphic memory optimization | +| mem_optim_off | - | - | Disable memory / graphic memory optimization | | ir_optim | - | - | Enable analysis and optimization of calculation graph,including OP fusion, etc | + + +For the mode of using Python code to start the prediction service, the API of the above two parameters is as follows: + +RPC Service +``` +from paddle_serving_server import Server +server = Server() +... +server.set_memory_optimize(mem_optim) +server.set_ir_optimize(ir_optim) +... +``` + +HTTP Service +``` +from paddle_serving_server import WebService +class NewService(WebService): +... +new_service = NewService(name="new") +... +new_service.prepare_server(mem_optim=True, ir_optim=False) +... +``` diff --git a/doc/PERFORMANCE_OPTIM_CN.md b/doc/PERFORMANCE_OPTIM_CN.md index c35ea7a11c40ad2a5752d9add8fd8d9f8ddb2b64..2fa5bdef1bee1cdc9e9daceaf853403485a06b84 100644 --- a/doc/PERFORMANCE_OPTIM_CN.md +++ b/doc/PERFORMANCE_OPTIM_CN.md @@ -14,7 +14,33 @@ 性能优化相关参数： +Paddle Serving中默认开启内存/显存优化选项，可以减少对内存/显存的占用，通常不会对性能造成影响，如果需要关闭可以在命令行启动模式中使用--mem_optim_off。 +ir_optim可以优化计算图，提升推理速度，默认关闭，在命令行启动的模式中通过--ir_optim开启。 + | 参数 | 类型 | 默认值 | 含义 | | --------- | ---- | ------ | -------------------------------- | -| mem_optim | - | - | 开启内存/显存优化 | +| mem_optim_off | - | - | 关闭内存/显存优化 | | ir_optim | - | - | 开启计算图分析优化，包括OP融合等 | + + +对于使用Python代码启动预测服务的模式，以上两个参数的接口如下： +RPC服务 +``` +from paddle_serving_server import Server +server = Server() +... +server.set_memory_optimize(mem_optim) +server.set_ir_optimize(ir_optim) +... +``` + +HTTP服务 +``` +from paddle_serving_server import WebService +class NewService(WebService): +... +new_service = NewService(name="new") +... +new_service.prepare_server(mem_optim=True, ir_optim=False) +... +``` diff --git a/python/examples/imagenet/resnet50_web_service.py b/python/examples/imagenet/resnet50_web_service.py index 1dbb2be90da5192baf8b274e9dd3255179842f6d..e7d1914973f2aeb58a912f7d85e35f85718d7a9b 100644 --- a/python/examples/imagenet/resnet50_web_service.py +++ b/python/examples/imagenet/resnet50_web_service.py @@ -54,6 +54,7 @@ class ImageService(WebService): score_list = fetch_map["score"] result = {"label": [], "prob": []} for score in score_list: + score = score.tolist() max_score = max(score) result["label"].append(self.label_dict[score.index(max_score)] .strip().replace(",", "")) diff --git a/python/paddle_serving_server/serve.py b/python/paddle_serving_server/serve.py index 009a6ce00af2290b64716e211429385d09189831..704cf0304adf1ac647c244063c2b23049f92b221 100644 --- a/python/paddle_serving_server/serve.py +++ b/python/paddle_serving_server/serve.py @@ -40,7 +40,7 @@ def parse_args(): # pylint: disable=doc-string-missing parser.add_argument( "--device", type=str, default="cpu", help="Type of device") parser.add_argument( - "--mem_optim", + "--mem_optim_off", default=False, action="store_true", help="Memory optimize") @@ -68,7 +68,7 @@ def start_standard_model(): # pylint: disable=doc-string-missing port = args.port workdir = args.workdir device = args.device - mem_optim = args.mem_optim + mem_optim = args.mem_optim_off is False ir_optim = args.ir_optim max_body_size = args.max_body_size use_mkl = args.use_mkl diff --git a/python/paddle_serving_server/web_service.py b/python/paddle_serving_server/web_service.py index d9b9e3f1b1dcfa9502096d0eab4e3be61d2bbaa6..b0c1b79bda5041b4eca114d778a23d3a123c226e 100755 --- a/python/paddle_serving_server/web_service.py +++ b/python/paddle_serving_server/web_service.py @@ -41,6 +41,8 @@ class WebService(object): server = Server() server.set_op_sequence(op_seq_maker.get_op_sequence()) server.set_num_threads(16) + server.set_memory_optimize(self.mem_optim) + server.set_ir_optimize(self.ir_optim) server.load_model_config(self.model_config) server.prepare_server( workdir=self.workdir, port=self.port_list[0], device=self.device) @@ -55,12 +57,19 @@ class WebService(object): else: return False - def prepare_server(self, workdir="", port=9393, device="cpu"): + def prepare_server(self, + workdir="", + port=9393, + device="cpu", + mem_optim=True, + ir_optim=False): self.workdir = workdir self.port = port self.device = device default_port = 12000 self.port_list = [] + self.mem_optim = mem_optim + self.ir_optim = ir_optim for i in range(1000): if self.port_is_available(default_port + i): self.port_list.append(default_port + i) @@ -83,8 +92,6 @@ class WebService(object): if isinstance(feed, dict) and "fetch" in feed: del feed["fetch"] fetch_map = self.client.predict(feed=feed, fetch=fetch) - for key in fetch_map: - fetch_map[key] = fetch_map[key].tolist() result = self.postprocess( feed=request.json["feed"], fetch=fetch, fetch_map=fetch_map) result = {"result": result} @@ -128,4 +135,6 @@ class WebService(object): return feed, fetch def postprocess(self, feed=[], fetch=[], fetch_map=None): + for key in fetch_map: + fetch_map[key] = fetch_map[key].tolist() return fetch_map diff --git a/python/paddle_serving_server_gpu/__init__.py b/python/paddle_serving_server_gpu/__init__.py index 70e9d27ccdd802700893d7d6fbca90bf7b8852be..1d94bf3093e6d76b260f53acd0c799080627c0ab 100644 --- a/python/paddle_serving_server_gpu/__init__.py +++ b/python/paddle_serving_server_gpu/__init__.py @@ -41,7 +41,7 @@ from concurrent import futures def serve_args(): parser = argparse.ArgumentParser("serve") parser.add_argument( - "--thread", type=int, default=10, help="Concurrency of server") + "--thread", type=int, default=2, help="Concurrency of server") parser.add_argument( "--model", type=str, default="", help="Model for serving") parser.add_argument( @@ -57,7 +57,7 @@ def serve_args(): parser.add_argument( "--name", type=str, default="None", help="Default service name") parser.add_argument( - "--mem_optim", + "--mem_optim_off", default=False, action="store_true", help="Memory optimize") @@ -187,7 +187,7 @@ class Server(object): self.cube_config_fn = "cube.conf" self.workdir = "" self.max_concurrency = 0 - self.num_threads = 4 + self.num_threads = 2 self.port = 8080 self.reload_interval_s = 10 self.max_body_size = 64 * 1024 * 1024 diff --git a/python/paddle_serving_server_gpu/serve.py b/python/paddle_serving_server_gpu/serve.py index e26b32c2699d09b714b2658cafad0ae8c5138071..3b0941a97560f11a52808fc7e152419e2cec0ba0 100644 --- a/python/paddle_serving_server_gpu/serve.py +++ b/python/paddle_serving_server_gpu/serve.py @@ -34,7 +34,7 @@ def start_gpu_card_model(index, gpuid, args): # pylint: disable=doc-string-miss port = args.port + index thread_num = args.thread model = args.model - mem_optim = args.mem_optim + mem_optim = args.mem_optim_off is False ir_optim = args.ir_optim max_body_size = args.max_body_size use_multilang = args.use_multilang diff --git a/python/paddle_serving_server_gpu/web_service.py b/python/paddle_serving_server_gpu/web_service.py index 2f848b1bf707aba17213d75facba6dea186a2351..5e9fdf4f4fda84dfb7c4f598fae6cf2381c377ca 100644 --- a/python/paddle_serving_server_gpu/web_service.py +++ b/python/paddle_serving_server_gpu/web_service.py @@ -41,7 +41,9 @@ class WebService(object): workdir="conf", port=9292, gpuid=0, - thread_num=10): + thread_num=2, + mem_optim=True, + ir_optim=False): device = "gpu" if gpuid == -1: device = "cpu" @@ -58,6 +60,8 @@ class WebService(object): server = Server() server.set_op_sequence(op_seq_maker.get_op_sequence()) server.set_num_threads(thread_num) + server.set_memory_optimize(mem_optim) + server.set_ir_optimize(ir_optim) server.load_model_config(self.model_config) if gpuid >= 0: @@ -77,7 +81,13 @@ class WebService(object): else: return False - def prepare_server(self, workdir="", port=9393, device="gpu", gpuid=0): + def prepare_server(self, + workdir="", + port=9393, + device="gpu", + gpuid=0, + mem_optim=True, + ir_optim=False): self.workdir = workdir self.port = port self.device = device @@ -94,7 +104,12 @@ class WebService(object): # init cpu service self.rpc_service_list.append( self.default_rpc_service( - self.workdir, self.port_list[0], -1, thread_num=10)) + self.workdir, + self.port_list[0], + -1, + thread_num=2, + mem_optim=mem_optim, + ir_optim=ir_optim)) else: for i, gpuid in enumerate(self.gpus): self.rpc_service_list.append( @@ -102,7 +117,9 @@ class WebService(object): "{}_{}".format(self.workdir, i), self.port_list[i], gpuid, - thread_num=10)) + thread_num=2, + mem_optim=mem_optim, + ir_optim=ir_optim)) def _launch_web_service(self): gpu_num = len(self.gpus)