diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 2a476f63862cfa2a41853d129bd6764df5292d3f..b712729f6420d019f1c1b5599f8e85cfef1a3765 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -25,6 +25,7 @@ import importlib import paddle.dataset import six.moves.cPickle as pickle import glob +import paddle __all__ = [] @@ -95,16 +96,19 @@ def download(url, module_name, md5sum, save_name=None): chunk_size = 4096 total_length = int(total_length) total_iter = total_length / chunk_size + 1 - log_interval = total_iter / 20 if total_iter > 20 else 1 + log_interval = total_iter // 20 if total_iter > 20 else 1 log_index = 0 + bar = paddle.hapi.progressbar.ProgressBar( + total_iter, name='item') for data in r.iter_content(chunk_size=chunk_size): if six.PY2: data = six.b(data) f.write(data) log_index += 1 + bar.update(log_index, {}) if log_index % log_interval == 0: - sys.stderr.write(".") - sys.stdout.flush() + bar.update(log_index) + except Exception as e: # re-try continue diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py index 5f63a3169f8ac7a09283bed98cc6f96d1193064b..6ed33f4f960b402fc97f32342a54c1c9ffd6e889 100644 --- a/python/paddle/hapi/progressbar.py +++ b/python/paddle/hapi/progressbar.py @@ -33,7 +33,8 @@ class ProgressBar(object): width=30, verbose=1, start=True, - file=sys.stdout): + file=sys.stdout, + name='step'): self._num = num if isinstance(num, int) and num <= 0: raise TypeError('num should be None or integer (> 0)') @@ -47,6 +48,7 @@ class ProgressBar(object): if start: self._start = time.time() self._last_update = 0 + self.name = name self._dynamic_display = ( (hasattr(self.file, 'isatty') and @@ -74,7 +76,7 @@ class ProgressBar(object): self.file.flush() self._start = time.time() - def update(self, current_num, values=None): + def update(self, current_num, values={}): now = time.time() if current_num: @@ -83,11 +85,11 @@ class ProgressBar(object): time_per_unit = 0 if time_per_unit >= 1 or time_per_unit == 0: - fps = ' - %.0fs/%s' % (time_per_unit, 'step') + fps = ' - %.0fs/%s' % (time_per_unit, self.name) elif time_per_unit >= 1e-3: - fps = ' - %.0fms/%s' % (time_per_unit * 1e3, 'step') + fps = ' - %.0fms/%s' % (time_per_unit * 1e3, self.name) else: - fps = ' - %.0fus/%s' % (time_per_unit * 1e6, 'step') + fps = ' - %.0fus/%s' % (time_per_unit * 1e6, self.name) info = '' if self._verbose == 1: @@ -102,7 +104,7 @@ class ProgressBar(object): if self._num is not None: numdigits = int(np.log10(self._num)) + 1 - bar_chars = ('step %' + str(numdigits) + 'd/%d [') % ( + bar_chars = (self.name + ' %' + str(numdigits) + 'd/%d [') % ( current_num, self._num) prog = float(current_num) / self._num prog_width = int(self._width * prog) @@ -116,7 +118,7 @@ class ProgressBar(object): bar_chars += ('.' * (self._width - prog_width)) bar_chars += ']' else: - bar_chars = 'step %3d' % current_num + bar_chars = self.name + ' %3d' % current_num self._total_width = len(bar_chars) sys.stdout.write(bar_chars) @@ -162,10 +164,10 @@ class ProgressBar(object): elif self._verbose == 2 or self._verbose == 3: if self._num: numdigits = int(np.log10(self._num)) + 1 - count = ('step %' + str(numdigits) + 'd/%d') % (current_num, - self._num) + count = (self.name + ' %' + str(numdigits) + 'd/%d') % ( + current_num, self._num) else: - count = 'step %3d' % current_num + count = self.name + ' %3d' % current_num info = count + info for k, val in values: