Fix out of order

serve/main.py

@@ -38,13 +38,18 @@ def successResp(data):
 
 
 sendQ = Queue()
-recvQ = Queue()
+thread_local = threading.local()
 
 
 @app.route('/', methods=['POST'])
 def infer():
-    sendQ.put(request.json)
-    success, resp = recvQ.get()
+    recv_queue = getattr(thread_local, 'recv_queue', None)
+    if recv_queue is None:
+        thread_local.recv_queue = Queue()
+        recv_queue = thread_local.recv_queue
+
+    sendQ.put((request.json, recv_queue))
+    success, resp = recv_queue.get()
     if success:
         return successResp(resp)
     else:
@@ -60,7 +65,7 @@ def worker():
         inferer = paddle.inference.Inference(parameters=params, fileobj=topo_f)
 
     while True:
-        j = sendQ.get()
+        j, recv_queue = sendQ.get()
         try:
             feeding = {}
             d = []
@@ -70,9 +75,9 @@ def worker():
                 r = inferer.infer([d], feeding=feeding)
         except:
             trace = traceback.format_exc()
-            recvQ.put((False, trace))
+            recv_queue.put((False, trace))
             continue
-        recvQ.put((True, r.tolist()))
+        recv_queue.put((True, r.tolist()))
 
 
 if __name__ == '__main__':