diff --git a/paddlespeech/server/utils/audio_handler.py b/paddlespeech/server/utils/audio_handler.py
index add04156df1610d167208cce15133731968ad429..4df6513374a8efe72f53243254aac79ddb2698cf 100644
--- a/paddlespeech/server/utils/audio_handler.py
+++ b/paddlespeech/server/utils/audio_handler.py
@@ -167,11 +167,11 @@ class ASRWsAudioHandler:
                 await ws.send(chunk_data.tobytes())
                 msg = await ws.recv()
                 msg = json.loads(msg)
-
-                if self.punc_server and len(msg["result"]) > 0:
-                    msg["result"] = self.punc_server.run(msg["result"])
                 logger.info("client receive msg={}".format(msg))
-
+            #client start to punctuation restore
+            if self.punc_server and len(msg['result']) > 0:
+                msg["result"] = self.punc_server.run(msg["result"])
+                logger.info("client punctuation restored msg={}".format(msg))
             # 4. we must send finished signal to the server
             audio_info = json.dumps(
                 {