From e8261156839aa304b151b641c4e0a5c47343aa31 Mon Sep 17 00:00:00 2001
From: frankwhzhang <frankwhzhang@126.com>
Date: Tue, 4 Aug 2020 10:44:11 +0800
Subject: [PATCH] fix dataloader log print

---
 core/trainers/framework/runner.py | 33 +++++++++++++++++++++++--------
 models/multitask/mmoe/config.yaml | 18 ++++++++---------
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/core/trainers/framework/runner.py b/core/trainers/framework/runner.py
index f316327a..efc674dd 100644
--- a/core/trainers/framework/runner.py
+++ b/core/trainers/framework/runner.py
@@ -17,6 +17,7 @@ from __future__ import print_function
 import os
 import time
 import numpy as np
+from datetime import datetime
 import paddle.fluid as fluid
 
 from paddlerec.core.utils import envs
@@ -137,7 +138,14 @@ class RunnerBase(object):
 
         metrics_varnames = []
         metrics_format = []
-        metrics_format.append("{}: {{}}".format("batch"))
+
+        if context["is_infer"]:
+            metrics_format.append("{{}}\t[Infer]\t{}: {{}}".format("batch"))
+        else:
+            metrics_format.append("{{}}\t[Train]\t{}: {{}}".format("batch"))
+
+        metrics_format.append("{}: {{:.2f}}s".format("time_each_interval"))
+
         for name, var in metrics.items():
             metrics_varnames.append(var.name)
             metrics_format.append("{}: {{}}".format(name))
@@ -146,6 +154,7 @@ class RunnerBase(object):
         reader = context["model"][model_dict["name"]]["model"]._data_loader
         reader.start()
         batch_id = 0
+        begin_time = time.time()
         scope = context["model"][model_name]["scope"]
         with fluid.scope_guard(scope):
             try:
@@ -154,15 +163,23 @@ class RunnerBase(object):
                         program=program,
                         fetch_list=metrics_varnames,
                         return_numpy=False)
-                    metrics = [batch_id]
-
-                    metrics_rets = [
-                        as_numpy(metrics_tensor)
-                        for metrics_tensor in metrics_tensors
-                    ]
-                    metrics.extend(metrics_rets)
 
                     if batch_id % fetch_period == 0 and batch_id != 0:
+                        metrics = [
+                            datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                        ]
+                        metrics.extend([batch_id])
+                        end_time = time.time()
+                        seconds = end_time - begin_time
+                        metrics.extend([seconds])
+                        begin_time = end_time
+
+                        metrics_rets = [
+                            as_numpy(metrics_tensor)
+                            for metrics_tensor in metrics_tensors
+                        ]
+                        metrics.extend(metrics_rets)
+
                         print(metrics_format.format(*metrics))
                     batch_id += 1
             except fluid.core.EOFException:
diff --git a/models/multitask/mmoe/config.yaml b/models/multitask/mmoe/config.yaml
index 63f052be..3dec64e1 100644
--- a/models/multitask/mmoe/config.yaml
+++ b/models/multitask/mmoe/config.yaml
@@ -17,12 +17,12 @@ workspace: "paddlerec.models.multitask.mmoe"
 dataset:
 - name: dataset_train
   batch_size: 1
-  type: QueueDataset
+  type: DataLoader # or QueueDataset
   data_path: "{workspace}/data/train"
   data_converter: "{workspace}/census_reader.py"
 - name: dataset_infer
   batch_size: 1
-  type: QueueDataset
+  type: DataLoader # or QueueDataset
   data_path: "{workspace}/data/train"
   data_converter: "{workspace}/census_reader.py"
 
@@ -37,9 +37,7 @@ hyper_parameters:
     learning_rate: 0.001
     strategy: async
 
-#use infer_runner mode and modify 'phase' below if infer
-mode: train_runner
-#mode: infer_runner
+mode: [train_runner, infer_runner]
 
 runner:
 - name: train_runner
@@ -53,7 +51,7 @@ runner:
   print_interval: 10
 - name: infer_runner
   class: infer
-  init_model_path: "increment/0"
+  init_model_path: "increment/1"
   device: cpu
 
 phase:
@@ -61,7 +59,7 @@ phase:
   model: "{workspace}/model.py"
   dataset_name: dataset_train
   thread_num: 1
-    #- name: infer
-    #  model: "{workspace}/model.py"
-    #  dataset_name: dataset_infer
-    #  thread_num: 1
+- name: infer
+  model: "{workspace}/model.py"
+  dataset_name: dataset_infer
+  thread_num: 1
-- 
GitLab