diff --git a/python/examples/bert/bert_web_service.py b/python/examples/bert/bert_web_service.py index e22e379d67e076d4712c8971b6d342b4eaceadb2..f72694c0e8c5bb7ab2778278d3fc79f13516dc12 100644 --- a/python/examples/bert/bert_web_service.py +++ b/python/examples/bert/bert_web_service.py @@ -36,3 +36,4 @@ bert_service.set_gpus(gpu_ids) bert_service.prepare_server( workdir="workdir", port=int(sys.argv[2]), device="gpu") bert_service.run_server() +bert_service.run_flask() diff --git a/python/examples/imagenet/image_classification_service.py b/python/examples/imagenet/image_classification_service.py index 2776eb1bc7126fab32dbb05774fb0060506b61af..ee3ae6dd1c64bda154bbadabe8d1e91da734fb5a 100644 --- a/python/examples/imagenet/image_classification_service.py +++ b/python/examples/imagenet/image_classification_service.py @@ -31,14 +31,14 @@ class ImageService(WebService): sample = base64.b64decode(image) img = reader.process_image(sample) res_feed = {} - res_feed["image"] = img.reshape(-1) + res_feed["image"] = img feed_batch.append(res_feed) return feed_batch, fetch else: sample = base64.b64decode(feed["image"]) img = reader.process_image(sample) res_feed = {} - res_feed["image"] = img.reshape(-1) + res_feed["image"] = img return res_feed, fetch @@ -47,3 +47,4 @@ image_service.load_model_config(sys.argv[1]) image_service.prepare_server( workdir=sys.argv[2], port=int(sys.argv[3]), device="cpu") image_service.run_server() +image_service.run_flask() diff --git a/python/examples/imagenet/image_classification_service_gpu.py b/python/examples/imagenet/image_classification_service_gpu.py index 287392e4f3ea922686cb03a032ba0b8e13d39709..d8ba4ed8cda9f600fb6d33441b90accdf5ecc532 100644 --- a/python/examples/imagenet/image_classification_service_gpu.py +++ b/python/examples/imagenet/image_classification_service_gpu.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle_serving_server_gpu.web_service import WebService import sys import cv2 import base64 import numpy as np from image_reader import ImageReader +from paddle_serving_server_gpu.web_service import WebService class ImageService(WebService): @@ -32,14 +32,14 @@ class ImageService(WebService): sample = base64.b64decode(image) img = reader.process_image(sample) res_feed = {} - res_feed["image"] = img.reshape(-1) + res_feed["image"] = img feed_batch.append(res_feed) return feed_batch, fetch else: sample = base64.b64decode(feed["image"]) img = reader.process_image(sample) res_feed = {} - res_feed["image"] = img.reshape(-1) + res_feed["image"] = img return res_feed, fetch @@ -49,3 +49,4 @@ image_service.set_gpus("0,1") image_service.prepare_server( workdir=sys.argv[2], port=int(sys.argv[3]), device="gpu") image_service.run_server() +image_service.run_flask() diff --git a/python/examples/imagenet/image_http_client.py b/python/examples/imagenet/image_http_client.py index cda0f33ac82d0bd228a22a8f438cbe1aa013eadf..d920eccb06cc9ad4a87237792a1e688fd76b0d6e 100644 --- a/python/examples/imagenet/image_http_client.py +++ b/python/examples/imagenet/image_http_client.py @@ -31,7 +31,7 @@ def predict(image_path, server): r = requests.post( server, data=req, headers={"Content-Type": "application/json"}) try: - print(r.json()["score"][0]) + print(r.json()["result"]["score"]) except ValueError: print(r.text) return r diff --git a/python/examples/imagenet/image_rpc_client.py b/python/examples/imagenet/image_rpc_client.py index 76f3a043474bf75e1e96a44f18ac7dfe3da11f78..f905179629f0dfc8c9da09b0cae90bae7be3687e 100644 --- a/python/examples/imagenet/image_rpc_client.py +++ b/python/examples/imagenet/image_rpc_client.py @@ -26,7 +26,7 @@ start = time.time() for i in range(1000): with open("./data/n01440764_10026.JPEG", "rb") as f: img = f.read() - img = reader.process_image(img).reshape(-1) + img = reader.process_image(img) fetch_map = client.predict(feed={"image": img}, fetch=["score"]) end = time.time() print(end - start) diff --git a/python/examples/imdb/text_classify_service.py b/python/examples/imdb/text_classify_service.py index 50d0d1aebba34a630c16442c6e3d00460bb1bc6a..5ff919ebb44b9a2590b148e4ccf8b91ce85f3f53 100755 --- a/python/examples/imdb/text_classify_service.py +++ b/python/examples/imdb/text_classify_service.py @@ -39,3 +39,4 @@ imdb_service.prepare_server( workdir=sys.argv[2], port=int(sys.argv[3]), device="cpu") imdb_service.prepare_dict({"dict_file_path": sys.argv[4]}) imdb_service.run_server() +imdb_service.run_flask() diff --git a/python/paddle_serving_server/__init__.py b/python/paddle_serving_server/__init__.py index 8062a7c83d99c0bed712ff46840b81f4557a353d..a58fb11ac3ee1fbe5086ae4381f6d6208c0c73ec 100644 --- a/python/paddle_serving_server/__init__.py +++ b/python/paddle_serving_server/__init__.py @@ -351,6 +351,7 @@ class Server(object): self._prepare_resource(workdir) self._prepare_engine(self.model_config_paths, device) self._prepare_infer_service(port) + self.port = port self.workdir = workdir infer_service_fn = "{}/{}".format(workdir, self.infer_service_fn) diff --git a/python/paddle_serving_server/web_service.py b/python/paddle_serving_server/web_service.py index 4a033cbcf1d32a55eaacbe9c0f6704e304e127b3..a03649725b1c41ca94b8ef495a2fc80e8293aba0 100755 --- a/python/paddle_serving_server/web_service.py +++ b/python/paddle_serving_server/web_service.py @@ -18,6 +18,8 @@ from flask import Flask, request, abort from multiprocessing import Pool, Process from paddle_serving_server import OpMaker, OpSeqMaker, Server from paddle_serving_client import Client +from contextlib import closing +import socket class WebService(object): @@ -41,19 +43,34 @@ class WebService(object): server.set_num_threads(16) server.load_model_config(self.model_config) server.prepare_server( - workdir=self.workdir, port=self.port + 1, device=self.device) + workdir=self.workdir, port=self.port_list[0], device=self.device) server.run_server() + def port_is_available(self, port): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(2) + result = sock.connect_ex(('0.0.0.0', port)) + if result != 0: + return True + else: + return False + def prepare_server(self, workdir="", port=9393, device="cpu"): self.workdir = workdir self.port = port self.device = device + default_port = 12000 + self.port_list = [] + for i in range(1000): + if self.port_is_available(default_port + i): + self.port_list.append(default_port + i) + break def _launch_web_service(self): - self.client_service = Client() - self.client_service.load_client_config( - "{}/serving_server_conf.prototxt".format(self.model_config)) - self.client_service.connect(["0.0.0.0:{}".format(self.port + 1)]) + self.client = Client() + self.client.load_client_config("{}/serving_server_conf.prototxt".format( + self.model_config)) + self.client.connect(["0.0.0.0:{}".format(self.port_list[0])]) def get_prediction(self, request): if not request.json: @@ -64,12 +81,12 @@ class WebService(object): feed, fetch = self.preprocess(request.json, request.json["fetch"]) if isinstance(feed, dict) and "fetch" in feed: del feed["fetch"] - fetch_map = self.client_service.predict(feed=feed, fetch=fetch) - for key in fetch_map: - fetch_map[key] = fetch_map[key][0].tolist() - result = self.postprocess( + fetch_map = self.client.predict(feed=feed, fetch=fetch) + fetch_map = self.postprocess( feed=request.json, fetch=fetch, fetch_map=fetch_map) - result = {"result": result} + for key in fetch_map: + fetch_map[key] = fetch_map[key].tolist() + result = {"result": fetch_map} except ValueError: result = {"result": "Request Value Error"} return result @@ -83,6 +100,24 @@ class WebService(object): p_rpc = Process(target=self._launch_rpc_service) p_rpc.start() + def run_flask(self): + app_instance = Flask(__name__) + + @app_instance.before_first_request + def init(): + self._launch_web_service() + + service_name = "/" + self.name + "/prediction" + + @app_instance.route(service_name, methods=["POST"]) + def run(): + return self.get_prediction(request) + + app_instance.run(host="0.0.0.0", + port=self.port, + threaded=False, + processes=4) + def preprocess(self, feed={}, fetch=[]): return feed, fetch diff --git a/python/paddle_serving_server_gpu/web_service.py b/python/paddle_serving_server_gpu/web_service.py index cb833ba32b20edeb22efd5b772506d32e05e4497..eb1ecfd8faaf34a6bf2955af46d5a8cf09085ad7 100644 --- a/python/paddle_serving_server_gpu/web_service.py +++ b/python/paddle_serving_server_gpu/web_service.py @@ -14,14 +14,15 @@ # pylint: disable=doc-string-missing from flask import Flask, request, abort -from paddle_serving_server_gpu import OpMaker, OpSeqMaker, Server -import paddle_serving_server_gpu as serving +from contextlib import closing from multiprocessing import Pool, Process, Queue from paddle_serving_client import Client +from paddle_serving_server_gpu import OpMaker, OpSeqMaker, Server from paddle_serving_server_gpu.serve import start_multi_card - +import socket import sys import numpy as np +import paddle_serving_server_gpu as serving class WebService(object): @@ -67,22 +68,39 @@ class WebService(object): def _launch_rpc_service(self, service_idx): self.rpc_service_list[service_idx].run_server() + def port_is_available(self, port): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(2) + result = sock.connect_ex(('0.0.0.0', port)) + if result != 0: + return True + else: + return False + def prepare_server(self, workdir="", port=9393, device="gpu", gpuid=0): self.workdir = workdir self.port = port self.device = device self.gpuid = gpuid + self.port_list = [] + default_port = 12000 + for i in range(1000): + if self.port_is_available(default_port + i): + self.port_list.append(default_port + i) + if len(self.port_list) > len(self.gpus): + break + if len(self.gpus) == 0: # init cpu service self.rpc_service_list.append( self.default_rpc_service( - self.workdir, self.port + 1, -1, thread_num=10)) + self.workdir, self.port_list[0], -1, thread_num=10)) else: for i, gpuid in enumerate(self.gpus): self.rpc_service_list.append( self.default_rpc_service( "{}_{}".format(self.workdir, i), - self.port + 1 + i, + self.port_list[i], gpuid, thread_num=10)) @@ -94,9 +112,9 @@ class WebService(object): endpoints = "" if gpu_num > 0: for i in range(gpu_num): - endpoints += "127.0.0.1:{},".format(self.port + i + 1) + endpoints += "127.0.0.1:{},".format(self.port_list[i]) else: - endpoints = "127.0.0.1:{}".format(self.port + 1) + endpoints = "127.0.0.1:{}".format(self.port_list[0]) self.client.connect([endpoints]) def get_prediction(self, request): @@ -109,11 +127,11 @@ class WebService(object): if isinstance(feed, dict) and "fetch" in feed: del feed["fetch"] fetch_map = self.client.predict(feed=feed, fetch=fetch) - for key in fetch_map: - fetch_map[key] = fetch_map[key][0].tolist() - result = self.postprocess( + fetch_map = self.postprocess( feed=request.json, fetch=fetch, fetch_map=fetch_map) - result = {"result": result} + for key in fetch_map: + fetch_map[key] = fetch_map[key].tolist() + result = {"result": fetch_map} except ValueError: result = {"result": "Request Value Error"} return result @@ -131,6 +149,24 @@ class WebService(object): for p in server_pros: p.start() + def run_flask(self): + app_instance = Flask(__name__) + + @app_instance.before_first_request + def init(): + self._launch_web_service() + + service_name = "/" + self.name + "/prediction" + + @app_instance.route(service_name, methods=["POST"]) + def run(): + return self.get_prediction(request) + + app_instance.run(host="0.0.0.0", + port=self.port, + threaded=False, + processes=4) + def preprocess(self, feed={}, fetch=[]): return feed, fetch