From 5209b9a510141fbd6f8c31a4cde838dffeaa0e6f Mon Sep 17 00:00:00 2001 From: Yi Liu Date: Tue, 30 Jun 2020 14:57:44 +0800 Subject: [PATCH] cat log to stdout when setting log_dir in launch (#25147) test=develop --- python/paddle/distributed/utils.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/utils.py b/python/paddle/distributed/utils.py index 511d501a22..53b02334cf 100644 --- a/python/paddle/distributed/utils.py +++ b/python/paddle/distributed/utils.py @@ -21,6 +21,7 @@ import signal import copy import sys import subprocess +import threading from contextlib import closing import socket @@ -370,7 +371,29 @@ def start_local_trainers(cluster, if log_dir is not None: os.system("mkdir -p {}".format(log_dir)) fn = open("%s/workerlog.%d" % (log_dir, idx), "a") - proc = subprocess.Popen(cmd, env=current_env, stdout=fn, stderr=fn) + if idx == 0: + proc = subprocess.Popen( + cmd, + env=current_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + def shell_tee(proc, fn): + BUF_SIZE = 512 + while True: + buf = proc.stdout.read(BUF_SIZE) + if len(buf) == 0: + break + + sys.stdout.buffer.write(buf) + fn.buffer.write(buf) + sys.stdout.flush() + fn.flush() + + threading.Thread(target=shell_tee, args=(proc, fn)).start() + else: + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn) else: proc = subprocess.Popen(cmd, env=current_env) -- GitLab