From c70f1cad1ba168cca83ad5c4c70f72c093fcdc1c Mon Sep 17 00:00:00 2001
From: LielinJiang <50691816+LielinJiang@users.noreply.github.com>
Date: Thu, 3 Jun 2021 16:44:44 +0800
Subject: [PATCH] Add progressbar for datasets downloading (#33302)

* add progressbar for datasets downloading
---
 python/paddle/dataset/common.py   | 10 +++++++---
 python/paddle/hapi/progressbar.py | 22 ++++++++++++----------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py
index 2a476f63862..b712729f642 100644
--- a/python/paddle/dataset/common.py
+++ b/python/paddle/dataset/common.py
@@ -25,6 +25,7 @@ import importlib
 import paddle.dataset
 import six.moves.cPickle as pickle
 import glob
+import paddle
 
 __all__ = []
 
@@ -95,16 +96,19 @@ def download(url, module_name, md5sum, save_name=None):
                     chunk_size = 4096
                     total_length = int(total_length)
                     total_iter = total_length / chunk_size + 1
-                    log_interval = total_iter / 20 if total_iter > 20 else 1
+                    log_interval = total_iter // 20 if total_iter > 20 else 1
                     log_index = 0
+                    bar = paddle.hapi.progressbar.ProgressBar(
+                        total_iter, name='item')
                     for data in r.iter_content(chunk_size=chunk_size):
                         if six.PY2:
                             data = six.b(data)
                         f.write(data)
                         log_index += 1
+                        bar.update(log_index, {})
                         if log_index % log_interval == 0:
-                            sys.stderr.write(".")
-                        sys.stdout.flush()
+                            bar.update(log_index)
+
         except Exception as e:
             # re-try
             continue
diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py
index 5f63a3169f8..6ed33f4f960 100644
--- a/python/paddle/hapi/progressbar.py
+++ b/python/paddle/hapi/progressbar.py
@@ -33,7 +33,8 @@ class ProgressBar(object):
                  width=30,
                  verbose=1,
                  start=True,
-                 file=sys.stdout):
+                 file=sys.stdout,
+                 name='step'):
         self._num = num
         if isinstance(num, int) and num <= 0:
             raise TypeError('num should be None or integer (> 0)')
@@ -47,6 +48,7 @@ class ProgressBar(object):
         if start:
             self._start = time.time()
         self._last_update = 0
+        self.name = name
 
         self._dynamic_display = (
             (hasattr(self.file, 'isatty') and
@@ -74,7 +76,7 @@ class ProgressBar(object):
         self.file.flush()
         self._start = time.time()
 
-    def update(self, current_num, values=None):
+    def update(self, current_num, values={}):
         now = time.time()
 
         if current_num:
@@ -83,11 +85,11 @@ class ProgressBar(object):
             time_per_unit = 0
 
         if time_per_unit >= 1 or time_per_unit == 0:
-            fps = ' - %.0fs/%s' % (time_per_unit, 'step')
+            fps = ' - %.0fs/%s' % (time_per_unit, self.name)
         elif time_per_unit >= 1e-3:
-            fps = ' - %.0fms/%s' % (time_per_unit * 1e3, 'step')
+            fps = ' - %.0fms/%s' % (time_per_unit * 1e3, self.name)
         else:
-            fps = ' - %.0fus/%s' % (time_per_unit * 1e6, 'step')
+            fps = ' - %.0fus/%s' % (time_per_unit * 1e6, self.name)
 
         info = ''
         if self._verbose == 1:
@@ -102,7 +104,7 @@ class ProgressBar(object):
             if self._num is not None:
                 numdigits = int(np.log10(self._num)) + 1
 
-                bar_chars = ('step %' + str(numdigits) + 'd/%d [') % (
+                bar_chars = (self.name + ' %' + str(numdigits) + 'd/%d [') % (
                     current_num, self._num)
                 prog = float(current_num) / self._num
                 prog_width = int(self._width * prog)
@@ -116,7 +118,7 @@ class ProgressBar(object):
                 bar_chars += ('.' * (self._width - prog_width))
                 bar_chars += ']'
             else:
-                bar_chars = 'step %3d' % current_num
+                bar_chars = self.name + ' %3d' % current_num
 
             self._total_width = len(bar_chars)
             sys.stdout.write(bar_chars)
@@ -162,10 +164,10 @@ class ProgressBar(object):
         elif self._verbose == 2 or self._verbose == 3:
             if self._num:
                 numdigits = int(np.log10(self._num)) + 1
-                count = ('step %' + str(numdigits) + 'd/%d') % (current_num,
-                                                                self._num)
+                count = (self.name + ' %' + str(numdigits) + 'd/%d') % (
+                    current_num, self._num)
             else:
-                count = 'step %3d' % current_num
+                count = self.name + ' %3d' % current_num
             info = count + info
 
             for k, val in values:
-- 
GitLab