未验证 提交 c70f1cad 编写于 作者: L LielinJiang 提交者: GitHub

Add progressbar for datasets downloading (#33302)

* add progressbar for datasets downloading
上级 273f3859
...@@ -25,6 +25,7 @@ import importlib ...@@ -25,6 +25,7 @@ import importlib
import paddle.dataset import paddle.dataset
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
import glob import glob
import paddle
__all__ = [] __all__ = []
...@@ -95,16 +96,19 @@ def download(url, module_name, md5sum, save_name=None): ...@@ -95,16 +96,19 @@ def download(url, module_name, md5sum, save_name=None):
chunk_size = 4096 chunk_size = 4096
total_length = int(total_length) total_length = int(total_length)
total_iter = total_length / chunk_size + 1 total_iter = total_length / chunk_size + 1
log_interval = total_iter / 20 if total_iter > 20 else 1 log_interval = total_iter // 20 if total_iter > 20 else 1
log_index = 0 log_index = 0
bar = paddle.hapi.progressbar.ProgressBar(
total_iter, name='item')
for data in r.iter_content(chunk_size=chunk_size): for data in r.iter_content(chunk_size=chunk_size):
if six.PY2: if six.PY2:
data = six.b(data) data = six.b(data)
f.write(data) f.write(data)
log_index += 1 log_index += 1
bar.update(log_index, {})
if log_index % log_interval == 0: if log_index % log_interval == 0:
sys.stderr.write(".") bar.update(log_index)
sys.stdout.flush()
except Exception as e: except Exception as e:
# re-try # re-try
continue continue
......
...@@ -33,7 +33,8 @@ class ProgressBar(object): ...@@ -33,7 +33,8 @@ class ProgressBar(object):
width=30, width=30,
verbose=1, verbose=1,
start=True, start=True,
file=sys.stdout): file=sys.stdout,
name='step'):
self._num = num self._num = num
if isinstance(num, int) and num <= 0: if isinstance(num, int) and num <= 0:
raise TypeError('num should be None or integer (> 0)') raise TypeError('num should be None or integer (> 0)')
...@@ -47,6 +48,7 @@ class ProgressBar(object): ...@@ -47,6 +48,7 @@ class ProgressBar(object):
if start: if start:
self._start = time.time() self._start = time.time()
self._last_update = 0 self._last_update = 0
self.name = name
self._dynamic_display = ( self._dynamic_display = (
(hasattr(self.file, 'isatty') and (hasattr(self.file, 'isatty') and
...@@ -74,7 +76,7 @@ class ProgressBar(object): ...@@ -74,7 +76,7 @@ class ProgressBar(object):
self.file.flush() self.file.flush()
self._start = time.time() self._start = time.time()
def update(self, current_num, values=None): def update(self, current_num, values={}):
now = time.time() now = time.time()
if current_num: if current_num:
...@@ -83,11 +85,11 @@ class ProgressBar(object): ...@@ -83,11 +85,11 @@ class ProgressBar(object):
time_per_unit = 0 time_per_unit = 0
if time_per_unit >= 1 or time_per_unit == 0: if time_per_unit >= 1 or time_per_unit == 0:
fps = ' - %.0fs/%s' % (time_per_unit, 'step') fps = ' - %.0fs/%s' % (time_per_unit, self.name)
elif time_per_unit >= 1e-3: elif time_per_unit >= 1e-3:
fps = ' - %.0fms/%s' % (time_per_unit * 1e3, 'step') fps = ' - %.0fms/%s' % (time_per_unit * 1e3, self.name)
else: else:
fps = ' - %.0fus/%s' % (time_per_unit * 1e6, 'step') fps = ' - %.0fus/%s' % (time_per_unit * 1e6, self.name)
info = '' info = ''
if self._verbose == 1: if self._verbose == 1:
...@@ -102,7 +104,7 @@ class ProgressBar(object): ...@@ -102,7 +104,7 @@ class ProgressBar(object):
if self._num is not None: if self._num is not None:
numdigits = int(np.log10(self._num)) + 1 numdigits = int(np.log10(self._num)) + 1
bar_chars = ('step %' + str(numdigits) + 'd/%d [') % ( bar_chars = (self.name + ' %' + str(numdigits) + 'd/%d [') % (
current_num, self._num) current_num, self._num)
prog = float(current_num) / self._num prog = float(current_num) / self._num
prog_width = int(self._width * prog) prog_width = int(self._width * prog)
...@@ -116,7 +118,7 @@ class ProgressBar(object): ...@@ -116,7 +118,7 @@ class ProgressBar(object):
bar_chars += ('.' * (self._width - prog_width)) bar_chars += ('.' * (self._width - prog_width))
bar_chars += ']' bar_chars += ']'
else: else:
bar_chars = 'step %3d' % current_num bar_chars = self.name + ' %3d' % current_num
self._total_width = len(bar_chars) self._total_width = len(bar_chars)
sys.stdout.write(bar_chars) sys.stdout.write(bar_chars)
...@@ -162,10 +164,10 @@ class ProgressBar(object): ...@@ -162,10 +164,10 @@ class ProgressBar(object):
elif self._verbose == 2 or self._verbose == 3: elif self._verbose == 2 or self._verbose == 3:
if self._num: if self._num:
numdigits = int(np.log10(self._num)) + 1 numdigits = int(np.log10(self._num)) + 1
count = ('step %' + str(numdigits) + 'd/%d') % (current_num, count = (self.name + ' %' + str(numdigits) + 'd/%d') % (
self._num) current_num, self._num)
else: else:
count = 'step %3d' % current_num count = self.name + ' %3d' % current_num
info = count + info info = count + info
for k, val in values: for k, val in values:
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册