add multiple gpu card for server

python/examples/bert/bert_client.py

@@ -9,6 +9,9 @@ import time
 from paddlehub.common.logger import logger
 import socket
 from paddle_serving_client import Client
+from paddle_serving_client.utils import MultiThreadRunner
+from paddle_serving_client.utils import benchmark_args
+args = benchmark_args()
 
 _ver = sys.version_info
 is_py2 = (_ver[0] == 2)
@@ -122,36 +125,27 @@ class BertService():
         return fetch_map_batch
 
 
-def test():
+def single_func(idx, resource):
     bc = BertService(
         model_name='bert_chinese_L-12_H-768_A-12',
         max_seq_len=20,
         show_ids=False,
         do_lower_case=True)
-    server_addr = ["127.0.0.1:9292"]
     config_file = './serving_client_conf/serving_client_conf.prototxt'
     fetch = ["pooled_output"]
+    server_addr = [resource["endpoint"][idx]]
     bc.load_client(config_file, server_addr)
     batch_size = 1
-    batch = []
-    for line in sys.stdin:
-        if batch_size == 1:
-            result = bc.run_general([[line.strip()]], fetch)
-            print(result)
-        else:
-            if len(batch) < batch_size:
-                batch.append([line.strip()])
-            else:
-                result = bc.run_batch_general(batch, fetch)
-                batch = []
-                for r in result:
-                    print(r)
-    if len(batch) > 0:
-        result = bc.run_batch_general(batch, fetch)
-        batch = []
-        for r in result:
-            print(r)
-
+    start = time.time()
+    fin = open("data-c.txt")
+    for line in fin:
+        result = bc.run_general([[line.strip()]], fetch)
+    end = time.time()
+    return [[end - start]]
 
 if __name__ == '__main__':
-    test()
+    multi_thread_runner = MultiThreadRunner()
+    result = multi_thread_runner.run(single_func, args.thread, {"endpoint":["127.0.0.1:9494", "127.0.0.1:9495", "127.0.0.1:9496", "127.0.0.1:9497"]})
+
+
+

python/paddle_serving_server_gpu/__init__.py

@@ -217,6 +217,7 @@ class Server(object):
         if not self.check_port(port):
             raise SystemExit("Prot {} is already used".format(port))
 
+        self.set_port(port)
         self._prepare_resource(workdir)
         self._prepare_engine(self.model_config_path, device)
         self._prepare_infer_service(port)

python/paddle_serving_server_gpu/serve.py

@@ -37,20 +37,22 @@ def parse_args():
     parser.add_argument(
         "--device", type=str, default="gpu", help="Type of device")
     parser.add_argument(
-        "--gpu_ids", type=int, default=0, help="gpu ids")
+        "--gpu_ids", type=str, default="", help="gpu ids")
     return parser.parse_args()
 
 args = parse_args()
 
 def start_gpu_card_model(gpuid):
+    gpuid = int(gpuid)
     device = "gpu"
     port = args.port
     if gpuid == -1:
         device = "cpu"
+    elif gpuid >= 0:
         port = args.port + gpuid
     thread_num = args.thread
     model = args.model
-    workdir = args.workdir
+    workdir = "{}_{}".format(args.workdir, gpuid)
 
     if model == "":
         print("You must specify your serving model")
@@ -83,8 +85,8 @@ if __name__ == "__main__":
         start_gpu_card_model(-1)
     else:
         gpu_processes = []
-        for i, gpu_id in gpus:
-            p = Process(target=start_gpu_card_model, (i,))
+        for i, gpu_id in enumerate(gpus):
+            p = Process(target=start_gpu_card_model, args=(i,))
             gpu_processes.append(p)
         for p in gpu_processes:
             p.start()