ppdet log iter and log time smooth (#2921)

* add log iter argument

* ppdet log time smooth

* ppdet log eta

tools/train.py

@@ -20,6 +20,8 @@ import os
 import time
 import multiprocessing
 import numpy as np
+import datetime
+from collections import deque
 
 
 def set_paddle_flags(**kwargs):
@@ -55,13 +57,14 @@ logger = logging.getLogger(__name__)
 
 def main():
     cfg = load_config(FLAGS.config)
-
     if 'architecture' in cfg:
         main_arch = cfg.architecture
     else:
         raise ValueError("'architecture' not specified in config file.")
 
     merge_config(FLAGS.opt)
+    if 'log_iter' not in cfg:
+        cfg.log_iter = 20
 
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
@@ -160,16 +163,22 @@ def main():
 
     cfg_name = os.path.basename(FLAGS.config).split('.')[0]
     save_dir = os.path.join(cfg.save_dir, cfg_name)
+    time_stat = deque(maxlen=cfg.log_iter)
     for it in range(start_iter, cfg.max_iters):
         start_time = end_time
         end_time = time.time()
+        time_stat.append(end_time - start_time)
+        time_cost = np.mean(time_stat)
+        eta_sec = (cfg.max_iters - it) * time_cost
+        eta = str(datetime.timedelta(seconds=int(eta_sec)))
         outs = exe.run(train_compile_program, fetch_list=train_values)
         stats = {k: np.array(v).mean() for k, v in zip(train_keys, outs[:-1])}
         train_stats.update(stats)
         logs = train_stats.log()
-        strs = 'iter: {}, lr: {:.6f}, {}, time: {:.3f}'.format(
-            it, np.mean(outs[-1]), logs, end_time - start_time)
-        logger.info(strs)
+        if it % cfg.log_iter == 0:
+            strs = 'iter: {}, lr: {:.6f}, {}, time: {:.3f}, eta: {}'.format(
+                it, np.mean(outs[-1]), logs, time_cost, eta)
+            logger.info(strs)
 
         if it > 0 and it % cfg.snapshot_iter == 0 or it == cfg.max_iters - 1:
             save_name = str(it) if it != cfg.max_iters - 1 else "model_final"