From 28f7b25bd83aebf202130fe986624daab208843a Mon Sep 17 00:00:00 2001 From: wuzewu Date: Thu, 14 Mar 2019 21:10:09 +0800 Subject: [PATCH] update commands --- paddle_hub/commands/__init__.py | 10 +++++- paddle_hub/commands/base_command.py | 15 +++++++-- paddle_hub/commands/download.py | 22 ++++++++++--- paddle_hub/commands/help.py | 26 +++++++++++---- paddle_hub/commands/hub.py | 12 ++++--- paddle_hub/commands/install.py | 46 ++++++++++++++++++++++++++ paddle_hub/commands/list.py | 43 ++++++++++++++++++++++++ paddle_hub/commands/run.py | 8 +++-- paddle_hub/commands/show.py | 29 +++++++++++++--- paddle_hub/commands/uninstall.py | 37 +++++++++++++++++++++ paddle_hub/commands/version.py | 12 ++++--- paddle_hub/module/manager.py | 51 +++++++++++++++++++++-------- paddle_hub/tools/downloader.py | 4 ++- 13 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 paddle_hub/commands/install.py create mode 100644 paddle_hub/commands/list.py create mode 100644 paddle_hub/commands/uninstall.py diff --git a/paddle_hub/commands/__init__.py b/paddle_hub/commands/__init__.py index 4778ecfb..923c6c6d 100644 --- a/paddle_hub/commands/__init__.py +++ b/paddle_hub/commands/__init__.py @@ -12,4 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import * +from . import hub +from . import download +from . import run +from . import show +from . import version +from . import list +from . import install +from . import uninstall +from . import help diff --git a/paddle_hub/commands/base_command.py b/paddle_hub/commands/base_command.py index 149d3fe7..538693be 100644 --- a/paddle_hub/commands/base_command.py +++ b/paddle_hub/commands/base_command.py @@ -21,18 +21,29 @@ import argparse class BaseCommand: + command_dict = {} + @classmethod def instance(cls): + if cls.name in BaseCommand.command_dict: + command = BaseCommand.command_dict[cls.name] + assert command.__class__.__name__ == cls.__name__, "already has a command %s with type %s" % ( + cls.name, command.__class__) + return command if not hasattr(cls, '_instance'): - cls._instance = cls() + cls._instance = cls(cls.name) + BaseCommand.command_dict[cls.name] = cls._instance return cls._instance - def __init__(self): + def __init__(self, name): assert not hasattr( self.__class__, '_instance'), 'Please use `instance()` to get Command object!' self.parser = argparse.ArgumentParser(description=__doc__) self.args = None + self.name = name + self.show_in_help = True + self.description = "" def add_arg(self, argument, type="str", default=None, help=None): add_argument( diff --git a/paddle_hub/commands/download.py b/paddle_hub/commands/download.py index 226ab27c..a02dcc66 100644 --- a/paddle_hub/commands/download.py +++ b/paddle_hub/commands/download.py @@ -23,8 +23,12 @@ from paddle_hub.module.manager import default_manager class DownloadCommand(BaseCommand): - def __init__(self): - super(DownloadCommand, self).__init__() + name = "download" + + def __init__(self, name): + super(DownloadCommand, self).__init__(name) + self.show_in_help = True + self.description = "Download a paddle hub module." # yapf: disable self.add_arg('--output_path', str, ".", "path to save the module, default in current directory" ) self.add_arg('--uncompress', bool, False, "uncompress the download package or not" ) @@ -35,13 +39,23 @@ class DownloadCommand(BaseCommand): def exec(self, argv): module_name = argv[1] + module_version = None if "==" not in module_name else module_name.split( + "==")[1] + module_name = module_name if "==" not in module_name else module_name.split( + "==")[0] self.args = self.parser.parse_args(argv[2:]) if not self.args.output_path: self.args.output_path = "." utils.check_path(self.args.output_path) - url = default_downloader.get_module_url(module_name) - assert url, "can't found module %s" % module_name + url = default_downloader.get_module_url( + module_name, version=module_version) + if not url: + tips = "can't found module %s" % module_name + if module_version: + tips += " with version %s" % module_version + print(tips) + return self.print_args() if self.args.uncompress: diff --git a/paddle_hub/commands/help.py b/paddle_hub/commands/help.py index 53c59240..9922dde8 100644 --- a/paddle_hub/commands/help.py +++ b/paddle_hub/commands/help.py @@ -15,19 +15,33 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from paddle_hub.tools.logger import logger from paddle_hub.commands.base_command import BaseCommand class HelpCommand(BaseCommand): - def __init__(self): - super(HelpCommand, self).__init__() + name = "help" - def help(self): - pass + def __init__(self, name): + super(HelpCommand, self).__init__(name) + self.show_in_help = True + self.description = "Show help for commands." + + def get_all_commands(self): + return BaseCommand.command_dict def exec(self, argv): - pass + hub_command = BaseCommand.command_dict["hub"] + help_text = "\n" + help_text += "Usage:\n" + help_text += "%s [options]\n" % hub_command.name + help_text += "\n" + help_text += "Commands:\n" + for command_name, command in self.get_all_commands().items(): + if not command.show_in_help or not command.description: + continue + help_text += " %-15s\t\t%s\n" % (command.name, command.description) + + print(help_text) command = HelpCommand.instance() diff --git a/paddle_hub/commands/hub.py b/paddle_hub/commands/hub.py index 76a21581..a90c3f55 100644 --- a/paddle_hub/commands/hub.py +++ b/paddle_hub/commands/hub.py @@ -26,8 +26,11 @@ import sys class HubCommand(BaseCommand): - def __init__(self): - super(HubCommand, self).__init__() + name = "hub" + + def __init__(self, name): + super(HubCommand, self).__init__(name) + self.show_in_help = False # yapf: disable self.add_arg('command', str, None, "command to run" ) # yapf: enable @@ -51,6 +54,7 @@ class HubCommand(BaseCommand): command(argv[2:]) +command = HubCommand.instance() + if __name__ == "__main__": - hub_command = HubCommand.instance() - hub_command.exec(sys.argv) + command.exec(sys.argv) diff --git a/paddle_hub/commands/install.py b/paddle_hub/commands/install.py new file mode 100644 index 00000000..72bff34d --- /dev/null +++ b/paddle_hub/commands/install.py @@ -0,0 +1,46 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from paddle_hub.tools.logger import logger +from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.tools import utils +from paddle_hub.module.manager import default_manager + + +class InstallCommand(BaseCommand): + name = "install" + + def __init__(self, name): + super(InstallCommand, self).__init__(name) + self.show_in_help = True + self.description = "Install the specify module to current environment." + #TODO(wuzewu): add --upgrade option + + def help(self): + self.parser.print_help() + + def exec(self, argv): + module_name = argv[1] + module_version = None if "==" not in module_name else module_name.split( + "==")[1] + module_name = module_name if "==" not in module_name else module_name.split( + "==")[0] + default_manager.install_module( + module_name=module_name, module_version=module_version) + + +command = InstallCommand.instance() diff --git a/paddle_hub/commands/list.py b/paddle_hub/commands/list.py new file mode 100644 index 00000000..c52729d6 --- /dev/null +++ b/paddle_hub/commands/list.py @@ -0,0 +1,43 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from paddle_hub.tools.logger import logger +from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.tools import utils +from paddle_hub.tools.downloader import default_downloader +from paddle_hub.module.manager import default_manager + + +class ListCommand(BaseCommand): + name = "list" + + def __init__(self, name): + super(ListCommand, self).__init__(name) + self.show_in_help = True + self.description = "List all module install in current environment." + + def exec(self, argv): + all_modules = default_manager.all_modules() + list_text = "\n" + list_text += " %-20s\t\t%s\n" % ("ModuleName", "ModulePath") + list_text += " %-20s\t\t%s\n" % ("--", "--") + for module_name, module_dir in all_modules.items(): + list_text += " %-20s\t\t%s\n" % (module_name, module_dir) + print(list_text) + + +command = ListCommand.instance() diff --git a/paddle_hub/commands/run.py b/paddle_hub/commands/run.py index c7f37185..c834263f 100644 --- a/paddle_hub/commands/run.py +++ b/paddle_hub/commands/run.py @@ -24,8 +24,12 @@ from paddle_hub.tools.arg_helper import add_argument, print_arguments class RunCommand(BaseCommand): - def __init__(self): - super(RunCommand, self).__init__() + name = "run" + + def __init__(self, name): + super(RunCommand, self).__init__(name) + self.show_in_help = True + self.description = "Run the specify module" # yapf: disable self.add_arg('--config', str, None, "config file in yaml format" ) self.add_arg('--dataset', str, None, "dataset be used" ) diff --git a/paddle_hub/commands/show.py b/paddle_hub/commands/show.py index 1d3b8233..d5d7f3fe 100644 --- a/paddle_hub/commands/show.py +++ b/paddle_hub/commands/show.py @@ -17,17 +17,36 @@ from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.module.manager import default_manager +from paddle_hub.module.module import Module class ShowCommand(BaseCommand): - def __init__(self): - super(ShowCommand, self).__init__() + name = "show" - def help(self): - pass + def __init__(self, name): + super(ShowCommand, self).__init__(name) + self.show_in_help = True + self.description = "Show the specify module's info" def exec(self, argv): - pass + module_name = argv[1] + self.args = self.parser.parse_args(argv[2:]) + + module_dir = default_manager.search_module(module_name) + if not module_dir: + return + + module = Module(module_dir=module_dir) + show_text = "Name:%s\n" % module.name + show_text += "Version:%s\n" % module.version + show_text += "Summary:\n" + show_text += " %s\n" % module.summary + show_text += "Author:%s\n" % module.author + show_text += "Author-Email:%s\n" % module.author_email + show_text += "Location:%s\n" % module_dir + #TODO(wuzewu): add more signature info + print(show_text) command = ShowCommand.instance() diff --git a/paddle_hub/commands/uninstall.py b/paddle_hub/commands/uninstall.py new file mode 100644 index 00000000..a7c0b2df --- /dev/null +++ b/paddle_hub/commands/uninstall.py @@ -0,0 +1,37 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from paddle_hub.tools.logger import logger +from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.tools import utils +from paddle_hub.module.manager import default_manager + + +class UninstallCommand(BaseCommand): + name = "uninstall" + + def __init__(self, name): + super(UninstallCommand, self).__init__(name) + self.show_in_help = True + self.description = "Uninstall the specify module from current environment." + + def exec(self, argv): + module_name = argv[1] + default_manager.uninstall_module(module_name=module_name) + + +command = UninstallCommand.instance() diff --git a/paddle_hub/commands/version.py b/paddle_hub/commands/version.py index 6b670f9c..70f63126 100644 --- a/paddle_hub/commands/version.py +++ b/paddle_hub/commands/version.py @@ -17,17 +17,19 @@ from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger from paddle_hub.commands.base_command import BaseCommand +from paddle_hub import version class VersionCommand(BaseCommand): - def __init__(self): - super(VersionCommand, self).__init__() + name = "version" - def help(self): - pass + def __init__(self, name): + super(VersionCommand, self).__init__(name) + self.show_in_help = True + self.description = "Get the paddle hub version" def exec(self, argv): - pass + print("hub %s" % version.hub_version) command = VersionCommand.instance() diff --git a/paddle_hub/module/manager.py b/paddle_hub/module/manager.py index f3935b47..7e727f4a 100644 --- a/paddle_hub/module/manager.py +++ b/paddle_hub/module/manager.py @@ -16,16 +16,17 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools import utils +from paddle_hub.tools.downloader import MODULE_HOME, default_downloader import os +import shutil class LocalModuleManager: def __init__(self, base_path=None): self.base_path = base_path if base_path else os.path.expanduser('~') utils.check_path(self.base_path) - self.local_hub_dir = os.path.join(self.base_path, ".hub") - self.local_modules_dir = os.path.join(self.local_hub_dir, "modules") - self.modules = [] + self.local_modules_dir = MODULE_HOME + self.modules_dict = {} if not os.path.exists(self.local_modules_dir): utils.mkdir(self.local_modules_dir) elif os.path.isfile(self.local_modules_dir): @@ -37,28 +38,50 @@ class LocalModuleManager: return True def all_modules(self, update=False): - if not update and self.modules: - return self.modules - self.modules = [] + if not update and self.modules_dict: + return self.modules_dict + self.modules_dict = {} for sub_dir_name in os.listdir(self.local_modules_dir): sub_dir_path = os.path.join(self.local_modules_dir, sub_dir_name) if os.path.isdir(sub_dir_path) and self.check_module_valid( sub_dir_path): #TODO(wuzewu): get module name - module_name = sub_dir_path - self.modules.append(module_name) + module_name = sub_dir_name + self.modules_dict[module_name] = sub_dir_path - return self.modules + return self.modules_dict def search_module(self, module_name, update=False): self.all_modules(update=update) - return module_name in self.all_modules + return self.modules_dict.get(module_name, None) - def install_module(self, upgrade=False): - pass + def install_module(self, module_name, module_version=None, upgrade=False): + self.all_modules(update=True) + if module_name in self.modules_dict: + module_dir = self.modules_dict[module_name] + print("module %s already install in %s" % (module_name, module_dir)) + return + url = default_downloader.get_module_url( + module_name, version=module_version) + #TODO(wuzewu): add compatibility check + if not url: + tips = "can't found module %s" % module_name + if module_version: + tips += " with version %s" % module_version + print(tips) + return - def uninstall_module(self): - pass + default_downloader.download_file_and_uncompress(url=url) + + def uninstall_module(self, module_name): + self.all_modules(update=True) + if not module_name in self.modules_dict: + print("%s is not installed" % module_name) + return + + module_dir = self.modules_dict[module_name] + shutil.rmtree(module_dir) + print("Successfully uninstalled %s" % module_name) default_manager = LocalModuleManager() diff --git a/paddle_hub/tools/downloader.py b/paddle_hub/tools/downloader.py index 11a75e2c..21984f47 100644 --- a/paddle_hub/tools/downloader.py +++ b/paddle_hub/tools/downloader.py @@ -30,7 +30,9 @@ from paddle_hub.data.reader import csv_reader __all__ = ['MODULE_HOME', 'downloader', 'md5file', 'Downloader'] # TODO(ZeyuChen) add environment varialble to set MODULE_HOME -MODULE_HOME = os.path.expanduser('~/.hub/module') +MODULE_HOME = os.path.expanduser('~') +MODULE_HOME = os.path.join(MODULE_HOME, ".hub") +MODULE_HOME = os.path.join(MODULE_HOME, "modules") # When running unit tests, there could be multiple processes that -- GitLab