From f82bea40fec402833a04ca4cd29a09bed2a8f509 Mon Sep 17 00:00:00 2001 From: tangwei Date: Fri, 1 May 2020 10:58:28 +0800 Subject: [PATCH] add log hint for local cluster training --- fleet_rec/core/engine/local_cluster_engine.py | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/fleet_rec/core/engine/local_cluster_engine.py b/fleet_rec/core/engine/local_cluster_engine.py index e6e108d1..3bc0bfed 100644 --- a/fleet_rec/core/engine/local_cluster_engine.py +++ b/fleet_rec/core/engine/local_cluster_engine.py @@ -53,13 +53,10 @@ class LocalClusterEngine(Engine): "POD_IP": user_endpoints_ips[i] }) - if logs_dir is not None: - os.system("mkdir -p {}".format(logs_dir)) - fn = open("%s/server.%d" % (logs_dir, i), "w") - log_fns.append(fn) - proc = subprocess.Popen(cmd, env=current_env, stdout=fn, stderr=fn, cwd=os.getcwd()) - else: - proc = subprocess.Popen(cmd, env=current_env, cwd=os.getcwd()) + os.system("mkdir -p {}".format(logs_dir)) + fn = open("%s/server.%d" % (logs_dir, i), "w") + log_fns.append(fn) + proc = subprocess.Popen(cmd, env=current_env, stdout=fn, stderr=fn, cwd=os.getcwd()) procs.append(proc) for i in range(worker_num): @@ -70,13 +67,10 @@ class LocalClusterEngine(Engine): "PADDLE_TRAINER_ID": str(i) }) - if logs_dir is not None: - os.system("mkdir -p {}".format(logs_dir)) - fn = open("%s/worker.%d" % (logs_dir, i), "w") - log_fns.append(fn) - proc = subprocess.Popen(cmd, env=current_env, stdout=fn, stderr=fn, cwd=os.getcwd()) - else: - proc = subprocess.Popen(cmd, env=current_env, cwd=os.getcwd()) + os.system("mkdir -p {}".format(logs_dir)) + fn = open("%s/worker.%d" % (logs_dir, i), "w") + log_fns.append(fn) + proc = subprocess.Popen(cmd, env=current_env, stdout=fn, stderr=fn, cwd=os.getcwd()) procs.append(proc) # only wait worker to finish here @@ -91,7 +85,9 @@ class LocalClusterEngine(Engine): if len(log_fns) > 0: log_fns[i].close() procs[i].terminate() - print("all workers and parameter servers already completed", file=sys.stderr) + print("all workers already completed, you can view logs under the {} directory".format(logs_dir), + file=sys.stderr) - def run(self): - self.start_procs() + +def run(self): + self.start_procs() -- GitLab