From f43c50a666d6e40ea10b3edcd35dc4280e96188e Mon Sep 17 00:00:00 2001 From: wuzewu Date: Mon, 2 Nov 2020 19:45:44 +0800 Subject: [PATCH] Update pypi utils --- paddlehub/module/manager.py | 44 +++++++++++++++++++++++++++---------- paddlehub/module/module.py | 3 ++- paddlehub/utils/pypi.py | 37 +++++++++++++++++++++---------- paddlehub/utils/utils.py | 6 ++++- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/paddlehub/module/manager.py b/paddlehub/module/manager.py index fbe10ce9..8eac13f3 100644 --- a/paddlehub/module/manager.py +++ b/paddlehub/module/manager.py @@ -156,8 +156,8 @@ class LocalModuleManager(object): branch: str = None) -> HubModule: ''' Install a HubModule from name or directory or archive file or url. When installing with the name parameter, if a - module that meets the conditions (both name and version) already installed, the installation step will be - skipped. When installing with other parameter, The locally installed modules will be uninstalled. + module that meets the conditions (both name and version) already installed, the installation step will be skipped. + When installing with other parameter, The locally installed modules will be uninstalled. Args: name (str|optional): module name to install @@ -278,7 +278,7 @@ class LocalModuleManager(object): def _install_from_source(self, name: str, version: str, source: str, update: bool = False, branch: str = None) -> HubModule: - '''Install a HubModule from Git Repo''' + '''Install a HubModule from git repository''' result = module_server.search_module(name=name, source=source, version=version, update=update, branch=branch) for item in result: if item['name'] == name and item['version'].match(version): @@ -292,7 +292,7 @@ class LocalModuleManager(object): os.makedirs(installed_path) module_file = os.path.join(installed_path, 'module.py') - # Generate a module.py file to reference objects from Git Repo + # Generate a module.py file to reference objects from git repository with open(module_file, 'w') as file: file.write('import sys\n\n') file.write('sys.path.insert(0, \'{}\')\n'.format(item['path'])) @@ -305,11 +305,20 @@ class LocalModuleManager(object): file.write('branch: {}'.format(branch)) self._local_modules[name] = HubModule.load(installed_path) + module_file = sys.modules[self._local_modules[name].__module__].__file__ + requirements_file = os.path.join(os.path.dirname(module_file), 'requirements.txt') + if os.path.exists(requirements_file): + shutil.copy(requirements_file, installed_path) + + # Install python package requirements + self._install_module_requirements(self._local_modules[name]) + if version: log.logger.info('Successfully installed {}-{}'.format(name, version)) else: log.logger.info('Successfully installed {}'.format(name)) return self._local_modules[name] + raise HubModuleNotFoundError(name=name, version=version, source=source) def _install_from_directory(self, directory: str) -> HubModule: @@ -339,14 +348,8 @@ class LocalModuleManager(object): hub_module_cls = HubModule.load(self._get_normalized_path(hub_module_cls.name)) self._local_modules[hub_module_cls.name] = hub_module_cls - for py_req in hub_module_cls.get_py_requirements(): - log.logger.info('Installing dependent packages: {}'.format(py_req)) - result = pypi.install(py_req) - if result: - log.logger.info('Successfully installed {}'.format(py_req)) - else: - log.logger.info('Some errors occurred while installing {}'.format(py_req)) - + # Install python package requirements + self._install_module_requirements(hub_module_cls) log.logger.info('Successfully installed {}-{}'.format(hub_module_cls.name, hub_module_cls.version)) return hub_module_cls @@ -361,3 +364,20 @@ class LocalModuleManager(object): path = os.path.normpath(path) directory = os.path.join(_tdir, path.split(os.sep)[0]) return self._install_from_directory(directory) + + def _install_module_requirements(self, module: HubModule): + file = utils.get_record_file() + with open(file, 'a') as _stream: + + for py_req in module.get_py_requirements(): + if py_req.lstrip().rstrip() == '': + continue + + with log.logger.processing('Installing dependent packages {}'.format(py_req)): + result = pypi.install(py_req, ostream=_stream, estream=_stream) + if result: + log.logger.info('Successfully installed dependent packages {}'.format(py_req)) + else: + log.logger.warning( + 'Some errors occurred while installing dependent packages {}. Detailed error information can be found in the {}.' + .format(py_req, file)) diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index 64fc930e..d2729530 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -114,7 +114,7 @@ class RunModule(object): if not os.path.exists(req_file): return [] with open(req_file, 'r') as file: - return file.read() + return file.read().split('\n') @property def is_runnable(self) -> bool: @@ -149,6 +149,7 @@ class Module(object): ''' def __new__(cls, + *, name: str = None, directory: str = None, version: str = None, diff --git a/paddlehub/utils/pypi.py b/paddlehub/utils/pypi.py index 1fce82cc..a63fc30f 100644 --- a/paddlehub/utils/pypi.py +++ b/paddlehub/utils/pypi.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pip +import os +import subprocess from pip._internal.utils.misc import get_installed_distributions +from typing import IO from paddlehub.utils.utils import Version from paddlehub.utils.io import discard_oe, typein @@ -38,20 +40,33 @@ def check(package: str, version: str = '') -> bool: return pdict[package].match(version) -def install(package: str, version: str = '', upgrade=False) -> bool: +def install(package: str, version: str = '', upgrade: bool = False, ostream: IO = None, estream: IO = None) -> bool: '''Install the python package.''' - with discard_oe(): - cmds = ['install', '{}{}'.format(package, version)] - if upgrade: - cmds.append('--upgrade') - result = pip.main(cmds) + package = package.replace(' ', '') + if version: + package = '{}=={}'.format(package, version) + + cmd = 'pip install "{}"'.format(package) + + if upgrade: + cmd += ' --upgrade' + + result, content = subprocess.getstatusoutput(cmd) + if result: + estream.write(content) + else: + ostream.write(content) return result == 0 -def uninstall(package: str) -> bool: +def uninstall(package: str, ostream: IO = None, estream: IO = None) -> bool: '''Uninstall the python package.''' - with discard_oe(), typein('y'): + with typein('y'): # type in 'y' to confirm the uninstall operation - cmds = ['uninstall', '{}'.format(package)] - result = pip.main(cmds) + cmd = 'pip uninstall {}'.format(package) + result, content = subprocess.getstatusoutput(cmd) + if result: + estream.write(content) + else: + ostream.write(content) return result == 0 diff --git a/paddlehub/utils/utils.py b/paddlehub/utils/utils.py index b99cae93..4daa7a2a 100644 --- a/paddlehub/utils/utils.py +++ b/paddlehub/utils/utils.py @@ -282,7 +282,7 @@ def md5(text: str): def record(msg: str) -> str: '''Record the specified text into the PaddleHub log file witch will be automatically stored according to date.''' - logfile = os.path.join(hubenv.LOG_HOME, time.strftime('%Y%m%d.log')) + logfile = get_record_file() with open(logfile, 'a') as file: file.write('=' * 50 + '\n') file.write('Record at ' + time.strftime('%Y-%m-%d %H:%M:%S') + '\n') @@ -297,3 +297,7 @@ def record_exception(msg: str) -> str: tb = traceback.format_exc() file = record(tb) utils.log.logger.warning('{}. Detailed error information can be found in the {}.'.format(msg, file)) + + +def get_record_file(): + return os.path.join(hubenv.LOG_HOME, time.strftime('%Y%m%d.log')) -- GitLab