diff --git a/paddlehub/__init__.py b/paddlehub/__init__.py index 1e105e3171177ad42aec0c034f19e00263ae296d..b8f2ee653dc9720ab19485df8ab201c7f08ab00a 100644 --- a/paddlehub/__init__.py +++ b/paddlehub/__init__.py @@ -19,8 +19,10 @@ from easydict import EasyDict __version__ = '2.0.0-alpha0' +from paddlehub import env from paddlehub.config import config from paddlehub.utils import log, parser, utils +from paddlehub.utils import download as _download from paddlehub.utils.paddlex import download, ResourceNotFoundError from paddlehub.server import server_check from paddlehub.server.server_source import ServerConnectionError @@ -40,6 +42,8 @@ from paddlehub.compat.task.config import RunConfig from paddlehub.compat.task.text_generation_task import TextGenerationTask sys.modules['paddlehub.io.parser'] = parser +sys.modules['paddlehub.common.dir'] = env +sys.modules['paddlehub.common.downloader'] = _download sys.modules['paddlehub.common.logger'] = log sys.modules['paddlehub.common.paddle_helper'] = paddle_utils sys.modules['paddlehub.common.utils'] = utils diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index d4cae6cd7fca2c58faf8f8ceeccf4901a0b1c454..a9413a773e845f1b3ef657383422d480f9f4aa16 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -67,15 +67,7 @@ class RunModule(object): '''The base class of PaddleHub Module, users can inherit this class to implement to realize custom class.''' def __init__(self, *args, **kwargs): - # Avoid module being initialized multiple times - if '_is_initialize' in self.__dict__ and self._is_initialize: - return - super(RunModule, self).__init__() - _run_func_name = self._get_func_name(self.__class__, _module_runnable_func) - self._run_func = getattr(self, _run_func_name) if _run_func_name else None - self._serving_func_name = self._get_func_name(self.__class__, _module_serving_func) - self._is_initialize = True def _get_func_name(self, current_cls: Generic, module_func_dict: dict) -> Optional[str]: mod = current_cls.__module__ + '.' + current_cls.__name__ @@ -133,7 +125,7 @@ class RunModule(object): `hub run` command. ''' return True if self._run_func else False - + @property def serving_func_name(self): return self._get_func_name(self.__class__, _module_serving_func) @@ -343,4 +335,4 @@ def moduleinfo(name: str, wrap_cls._hook_by_hub = True return wrap_cls - return _wrapper \ No newline at end of file + return _wrapper diff --git a/paddlehub/utils/download.py b/paddlehub/utils/download.py index 87065b18ba2af9a4cba5586c741db70ec0ed420e..164f05d44795b42e5f0690ab3c29f20830322f85 100644 --- a/paddlehub/utils/download.py +++ b/paddlehub/utils/download.py @@ -15,18 +15,40 @@ import os -from paddlehub.env import DATA_HOME +import paddlehub.env as hubenv from paddle.utils.download import get_path_from_url +from paddlehub.utils import log, utils, xarfile def download_data(url): save_name = os.path.basename(url).split('.')[0] - output_path = os.path.join(DATA_HOME, save_name) + output_path = os.path.join(hubenv.DATA_HOME, save_name) if not os.path.exists(output_path): - get_path_from_url(url, DATA_HOME) + get_path_from_url(url, hubenv.DATA_HOME) def _wrapper(Dataset): return Dataset return _wrapper + + +class Downloader: + def download_file_and_uncompress(self, url: str, save_path: str, print_progress: bool): + with utils.generate_tempdir() as _dir: + if print_progress: + with log.ProgressBar('Download {}'.format(url)) as bar: + for path, ds, ts in utils.download_with_progress(url=url, path=_dir): + bar.update(float(ds) / ts) + else: + path = utils.download(url=url, path=_dir) + + if print_progress: + with log.ProgressBar('Decompress {}'.format(path)) as bar: + for path, ds, ts in xarfile.unarchive_with_progress(name=path, path=save_path): + bar.update(float(ds) / ts) + else: + path = xarfile.unarchive(name=path, path=save_path) + + +default_downloader = Downloader() diff --git a/paddlehub/utils/utils.py b/paddlehub/utils/utils.py index c5e45754dcb15845474fbaca7443e74fe553234d..1839b3dd06a8db78ea178b15168680f160014566 100644 --- a/paddlehub/utils/utils.py +++ b/paddlehub/utils/utils.py @@ -154,10 +154,12 @@ def seconds_to_hms(seconds: int) -> str: hms_str = '{:0>2}:{:0>2}:{:0>2}'.format(h, m, s) return hms_str + def cv2_to_base64(image: np.ndarray) -> str: data = cv2.imencode('.jpg', image)[1] return base64.b64encode(data.tostring()).decode('utf8') + def base64_to_cv2(b64str: str) -> np.ndarray: '''Convert a string in base64 format to cv2 data''' data = base64.b64decode(b64str.encode('utf8')) @@ -304,11 +306,11 @@ def record_exception(msg: str) -> str: utils.log.logger.warning('{}. Detailed error information can be found in the {}.'.format(msg, file)) -def get_record_file(): +def get_record_file() -> str: return os.path.join(hubenv.LOG_HOME, time.strftime('%Y%m%d.log')) -def is_port_occupied(ip, port): +def is_port_occupied(ip: str, port: int) -> bool: ''' Check if port os occupied. ''' @@ -319,3 +321,9 @@ def is_port_occupied(ip, port): return True except: return False + + +def mkdir(path: str): + """The same as the shell command `mkdir -p`.""" + if not os.path.exists(path): + os.makedirs(path)