add IO percent for multi_trainer

paddle/fluid/framework/downpour_worker.cc

@@ -211,13 +211,16 @@ void DownpourWorker::TrainFilesWithProfiler() {
                                      &feature_values_[tid], table.fea_dim());
       timeline.Pause();
       pull_sparse_time += timeline.ElapsedSec();
+      total_time += timeline.ElapsedSec();
       CollectLabelInfo(i);
       timeline.Pause();
       collect_label_time += timeline.ElapsedSec();
+      total_time += timeline.ElapsedSec();
       timeline.Start();
       FillSparseValue(i);
       timeline.Pause();
       fill_sparse_time += timeline.ElapsedSec();
+      total_time += timeline.ElapsedSec();
     }
     VLOG(3) << "Fill sparse value for all sparse table done.";
 
@@ -257,6 +260,7 @@ void DownpourWorker::TrainFilesWithProfiler() {
           &feature_grads_[tid], &push_sparse_status_);
       timeline.Pause();
       push_sparse_time += timeline.ElapsedSec();
+      total_time += timeline.ElapsedSec();
     }
 
     timeline.Start();
@@ -269,7 +273,7 @@ void DownpourWorker::TrainFilesWithProfiler() {
     }
     timeline.Pause();
     push_dense_time += timeline.ElapsedSec();
-
+    total_time += timeline.ElapsedSec();
     VLOG(3) << "push sparse and dense gradient done.";
     int32_t tmp_push_dense_wait_times = -1;
     int32_t tmp_push_sparse_wait_times = -1;
@@ -324,6 +328,7 @@ void DownpourWorker::TrainFilesWithProfiler() {
         fprintf(stderr, "IO percent: %f\n", read_time / total_time * 100);
       }
     }
+    timeline.Start();
   }
 }
 

paddle/fluid/framework/hogwild_worker.cc

@@ -110,12 +110,7 @@ void HogwildWorker::TrainFilesWithProfiler() {
                   op_name[i].c_str(), op_total_time[i] / batch_cnt);
         }
         fprintf(stderr, "mean read time: %fs\n", read_time / batch_cnt);
-        /*
-        int fetch_var_num = fetch_var_names_.size();
-        for (int i = 0; i < fetch_var_num; ++i) {
-          print_fetch_var(thread_scope_, fetch_var_names_[i]);
-        }
-        */
+        fprintf(stderr, "IO percent: %f\n", read_time / total_time * 100);
       }
     }
     timeline.Start();

python/paddle/fluid/executor.py

@@ -636,21 +636,28 @@ class Executor(object):
         if not compiled:
             trainer = TrainerFactory().create_trainer(program._fleet_opt)
             trainer.set_program(program)
-            with open("fleet_desc.prototxt", "w") as fout:
-                fout.write(str(program._fleet_opt["fleet_desc"]))
         else:
             trainer = TrainerFactory().create_trainer(
                 program.program._fleet_opt)
             trainer.set_program(program.program)
         if thread <= 0:
-            trainer.set_thread(dataset.thread_num)
+            if dataset.thread_num <= 0:
+                raise RuntimeError(
+                    "You should set thread num first, either in Dataset or in Executor.train_from_dataset"
+                )
+            else:
+                trainer.set_thread(dataset.thread_num)
         else:
             trainer.set_thread(thread)
         trainer.set_debug(debug)
         trainer.gen_trainer_desc()
         dataset._prepare_to_run()
-        with open("trainer_desc.prototxt", "w") as fout:
-            fout.write(trainer._desc())
+        if debug:
+            with open("train_desc.prototxt", "w") as fout:
+                fout.write(trainer._desc())
+            if program._fleet_opt:
+                with open("fleet_desc.prototxt", "w") as fout:
+                    fout.write(str(program._fleet_opt["fleet_desc"]))
         self._default_executor.run_from_dataset(program.desc, scope,
                                                 dataset.dataset,
                                                 trainer._desc())