diff --git a/.copyright.hook b/.copyright.hook new file mode 100644 index 0000000000000000000000000000000000000000..0871ae716a7fad16dccb7f7f6164f86c2db6b242 --- /dev/null +++ b/.copyright.hook @@ -0,0 +1,121 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import io, re +import sys, os +import subprocess +import platform + +COPYRIGHT = ''' + Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +LANG_COMMENT_MARK = None + +NEW_LINE_MARK = None + +COPYRIGHT_HEADER = None + +if platform.system() == "Windows": + NEW_LINE_MARK = "\r\n" +else: + NEW_LINE_MARK = '\n' + COPYRIGHT_HEADER = COPYRIGHT.split(NEW_LINE_MARK)[1] + p = re.search('(\d{4})', COPYRIGHT_HEADER).group(0) + process = subprocess.Popen(["date", "+%Y"], stdout=subprocess.PIPE) + date, err = process.communicate() + date = date.decode("utf-8").rstrip("\n") + COPYRIGHT_HEADER = COPYRIGHT_HEADER.replace(p, date) + + +def generate_copyright(template, lang='C'): + if lang == 'Python': + LANG_COMMENT_MARK = '#' + else: + LANG_COMMENT_MARK = "//" + + lines = template.split(NEW_LINE_MARK) + BLANK = " " + ans = LANG_COMMENT_MARK + BLANK + COPYRIGHT_HEADER + NEW_LINE_MARK + for lino, line in enumerate(lines): + if lino == 0 or lino == 1 or lino == len(lines) - 1: continue + if len(line) == 0: + BLANK = "" + else: + BLANK = " " + ans += LANG_COMMENT_MARK + BLANK + line + NEW_LINE_MARK + + return ans + "\n" + + +def lang_type(filename): + if filename.endswith(".py"): + return "Python" + elif filename.endswith(".h"): + return "C" + elif filename.endswith(".c"): + return "C" + elif filename.endswith(".hpp"): + return "C" + elif filename.endswith(".cc"): + return "C" + elif filename.endswith(".cpp"): + return "C" + elif filename.endswith(".cu"): + return "C" + elif filename.endswith(".cuh"): + return "C" + elif filename.endswith(".go"): + return "C" + elif filename.endswith(".proto"): + return "C" + else: + print("Unsupported filetype %s", filename) + exit(0) + + +PYTHON_ENCODE = re.compile("^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Checker for copyright declaration.') + parser.add_argument('filenames', nargs='*', help='Filenames to check') + args = parser.parse_args(argv) + + retv = 0 + for filename in args.filenames: + fd = io.open(filename, encoding="utf-8") + first_line = fd.readline() + second_line = fd.readline() + if "COPYRIGHT (C)" in first_line.upper(): continue + if first_line.startswith("#!") or PYTHON_ENCODE.match( + second_line) != None or PYTHON_ENCODE.match(first_line) != None: + continue + original_contents = io.open(filename, encoding="utf-8").read() + new_contents = generate_copyright( + COPYRIGHT, lang_type(filename)) + original_contents + print('Auto Insert Copyright Header {}'.format(filename)) + retv = 1 + with io.open(filename, 'w') as output_file: + output_file.write(new_contents) + + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/paddle_hub/commands/__init__.py b/paddle_hub/commands/__init__.py index eacb38f0f9fd59bdd9e7eec1d32fedbbfc130f3c..8ca5e72be20bab36834657b8d3f6bbe1d3697851 100644 --- a/paddle_hub/commands/__init__.py +++ b/paddle_hub/commands/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import hub from . import download from . import run from . import show @@ -22,3 +21,4 @@ from . import install from . import uninstall from . import search from . import help +from . import hub diff --git a/paddle_hub/commands/base_command.py b/paddle_hub/commands/base_command.py index 538693bee90f3534f853fe5f0ad05d03848e339b..49f164b175cd3faa00c5466037a7885971eb9e6e 100644 --- a/paddle_hub/commands/base_command.py +++ b/paddle_hub/commands/base_command.py @@ -16,9 +16,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.arg_helper import add_argument, print_arguments -from paddle_hub.tools.logger import logger import argparse +ENTRY = "hub" + class BaseCommand: command_dict = {} @@ -39,12 +40,14 @@ class BaseCommand: assert not hasattr( self.__class__, '_instance'), 'Please use `instance()` to get Command object!' - self.parser = argparse.ArgumentParser(description=__doc__) self.args = None self.name = name self.show_in_help = True self.description = "" + def help(self): + self.parser.print_help() + def add_arg(self, argument, type="str", default=None, help=None): add_argument( argument=argument, diff --git a/paddle_hub/commands/download.py b/paddle_hub/commands/download.py index 566a55d1440fba57f8c9d460d9cd122fcb9c0742..e8f5d8b25765d1ae89f3a60aac8646ab833ffa18 100644 --- a/paddle_hub/commands/download.py +++ b/paddle_hub/commands/download.py @@ -16,10 +16,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.tools import utils from paddle_hub.tools.downloader import default_downloader from paddle_hub.hub_server import default_hub_server +import argparse class DownloadCommand(BaseCommand): @@ -29,15 +30,21 @@ class DownloadCommand(BaseCommand): super(DownloadCommand, self).__init__(name) self.show_in_help = True self.description = "Download a paddle hub module." + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s [options]', + add_help=False) # yapf: disable - self.add_arg('--output_path', str, ".", "path to save the module, default in current directory" ) + self.add_arg('--output_path', str, ".", "path to save the module" ) self.add_arg('--uncompress', bool, False, "uncompress the download package or not" ) # yapf: enable - def help(self): - self.parser.print_help() - def exec(self, argv): + if not argv: + print("ERROR: Please specify a module\n") + self.help() + return False module_name = argv[0] module_version = None if "==" not in module_name else module_name.split( "==")[1] @@ -55,7 +62,7 @@ class DownloadCommand(BaseCommand): if module_version: tips += " with version %s" % module_version print(tips) - return + return True self.print_args() if self.args.uncompress: @@ -65,7 +72,7 @@ class DownloadCommand(BaseCommand): result, tips, file = default_downloader.download_file( url=url, save_path=self.args.output_path) print(tips) - return result + return True command = DownloadCommand.instance() diff --git a/paddle_hub/commands/help.py b/paddle_hub/commands/help.py index 9922dde801c30f844988ef629585b686eba6b907..11324f95278a3ce0e9ec30b21d940ae53c046dae 100644 --- a/paddle_hub/commands/help.py +++ b/paddle_hub/commands/help.py @@ -42,6 +42,7 @@ class HelpCommand(BaseCommand): help_text += " %-15s\t\t%s\n" % (command.name, command.description) print(help_text) + return True command = HelpCommand.instance() diff --git a/paddle_hub/commands/hub.py b/paddle_hub/commands/hub.py index 857b2de915008f5d1e5e836a589f64734d77075f..fb4b4c5edcc025b873e238e84ea39d55f96e59fa 100644 --- a/paddle_hub/commands/hub.py +++ b/paddle_hub/commands/hub.py @@ -31,22 +31,24 @@ class HubCommand(BaseCommand): def __init__(self, name): super(HubCommand, self).__init__(name) self.show_in_help = False - # yapf: disable - self.add_arg('command', str, None, "command to run" ) - # yapf: enable def exec(self, argv): - args = self.parser.parse_args(argv[1:2]) - - if not args.command in BaseCommand.command_dict: - logger.critical("command %s not supported!" % args.command) + if not argv: + help.command.exec(argv) + exit(1) + return False + sub_command = argv[0] + if not sub_command in BaseCommand.command_dict: + print("ERROR: unknown command '%s'" % sub_command) + help.command.exec(argv) exit(1) + return False - command = BaseCommand.command_dict[args.command] - command.exec(argv[2:]) + command = BaseCommand.command_dict[sub_command] + return command.exec(argv[1:]) command = HubCommand.instance() if __name__ == "__main__": - command.exec(sys.argv) + command.exec(sys.argv[1:]) diff --git a/paddle_hub/commands/install.py b/paddle_hub/commands/install.py index 51e62e0c164631d44410de09e358f15a71abcb94..4c48817aa920df2630c22a06dbc73f6441e691c3 100644 --- a/paddle_hub/commands/install.py +++ b/paddle_hub/commands/install.py @@ -16,9 +16,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.tools import utils from paddle_hub.module.manager import default_module_manager +import argparse class InstallCommand(BaseCommand): @@ -28,12 +29,18 @@ class InstallCommand(BaseCommand): super(InstallCommand, self).__init__(name) self.show_in_help = True self.description = "Install the specify module to current environment." + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s', + add_help=False) #TODO(wuzewu): add --upgrade option - def help(self): - self.parser.print_help() - def exec(self, argv): + if not argv: + print("ERROR: Please specify a module\n") + self.help() + return False module_name = argv[0] module_version = None if "==" not in module_name else module_name.split( "==")[1] @@ -42,7 +49,7 @@ class InstallCommand(BaseCommand): result, tips, module_dir = default_module_manager.install_module( module_name=module_name, module_version=module_version) print(tips) - return result + return True command = InstallCommand.instance() diff --git a/paddle_hub/commands/list.py b/paddle_hub/commands/list.py index 3c38b7840049cdb436844ddaf51cdf5b39dff53b..5c0dc41572874e37116732692f644468e8a5ff06 100644 --- a/paddle_hub/commands/list.py +++ b/paddle_hub/commands/list.py @@ -38,6 +38,7 @@ class ListCommand(BaseCommand): for module_name, module_dir in all_modules.items(): list_text += " %-20s\t\t%s\n" % (module_name, module_dir) print(list_text) + return True command = ListCommand.instance() diff --git a/paddle_hub/commands/run.py b/paddle_hub/commands/run.py index c3bb12192177b7becc1399ddb22419af4b688a88..96984f6a1969e002906e9e8195dc68c85fbf060b 100644 --- a/paddle_hub/commands/run.py +++ b/paddle_hub/commands/run.py @@ -15,13 +15,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function + from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand -import paddle_hub as hub +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.io.reader import csv_reader, yaml_reader +from paddle_hub.module.manager import default_module_manager from paddle_hub.tools import utils from paddle_hub.tools.arg_helper import add_argument, print_arguments +import paddle_hub as hub +import argparse +import os + class RunCommand(BaseCommand): name = "run" @@ -30,51 +35,59 @@ class RunCommand(BaseCommand): super(RunCommand, self).__init__(name) self.show_in_help = True self.description = "Run the specify module" + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s [options]') # yapf: disable self.add_arg('--config', str, None, "config file in yaml format" ) self.add_arg('--dataset', str, None, "dataset be used" ) - self.add_arg('--module', str, None, "module to run" ) self.add_arg('--signature', str, None, "signature to run" ) # yapf: enable - def _check_module(self): - if not self.args.module: - logger.critical("lack of module") - self.help() - exit(1) - def _check_dataset(self): if not self.args.dataset: - logger.critical("lack of dataset file") + print("Error! Lack of dataset file") self.help() exit(1) if not utils.is_csv_file(self.args.dataset): - logger.critical("dataset file should in csv format") + print("Error! Dataset file should in csv format") self.help() exit(1) def _check_config(self): if not self.args.config: - logger.critical("lack of config file") + print("Error! Lack of config file") self.help() exit(1) if not utils.is_yaml_file(self.args.config): - logger.critical("config file should in yaml format") + print("Error! Config file should in yaml format") self.help() exit(1) - def help(self): - self.parser.print_help() - def exec(self, argv): - self.args = self.parser.parse_args(argv) - self.print_args() - - self._check_module() + if not argv: + print("ERROR: Please specify a key\n") + self.help() + return False + module_name = argv[0] + self.args = self.parser.parse_args(argv[1:]) self._check_dataset() self._check_config() - module = hub.Module(module_dir=self.args.module) + module_dir = default_module_manager.search_module(module_name) + if not module_dir: + if os.path.exists(module_name): + module_dir = module_name + else: + print("Install Module %s" % module_name) + result, tips, module_dir = default_module_manager.install_module( + module_name) + print(tips) + if not result: + return False + + module = hub.Module(module_dir=module_dir) yaml_config = yaml_reader.read(self.args.config) if not self.args.signature: diff --git a/paddle_hub/commands/search.py b/paddle_hub/commands/search.py index dba7561fd71b0fa8205f714433f9484483d737a5..516efbda5653518defdd6b29e4d39f8693b503e0 100644 --- a/paddle_hub/commands/search.py +++ b/paddle_hub/commands/search.py @@ -16,9 +16,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.tools import utils from paddle_hub.hub_server import default_hub_server +import argparse class SearchCommand(BaseCommand): @@ -28,8 +29,18 @@ class SearchCommand(BaseCommand): super(SearchCommand, self).__init__(name) self.show_in_help = True self.description = "Search a paddle hub module with keyword." + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s', + add_help=False) def exec(self, argv): + if not argv: + print("ERROR: Please specify a key\n") + self.help() + return False + module_name = argv[0] module_list = default_hub_server.search_module(module_name) text = "\n" @@ -38,6 +49,7 @@ class SearchCommand(BaseCommand): for module_name, module_version in module_list: text += " %-20s\t\t%s\n" % (module_name, module_version) print(text) + return True command = SearchCommand.instance() diff --git a/paddle_hub/commands/show.py b/paddle_hub/commands/show.py index 5a928b2af5d7697cb3f67df5405a0b5435b27a73..eb7b178cfd6db46aee635a935cb6716dfea120ef 100644 --- a/paddle_hub/commands/show.py +++ b/paddle_hub/commands/show.py @@ -16,10 +16,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.module.manager import default_module_manager from paddle_hub.module.module import Module import os +import argparse class ShowCommand(BaseCommand): @@ -29,8 +30,18 @@ class ShowCommand(BaseCommand): super(ShowCommand, self).__init__(name) self.show_in_help = True self.description = "Show the specify module's info" + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s', + add_help=False) def exec(self, argv): + if not argv: + print("ERROR: Please specify a module\n") + self.help() + return False + module_name = argv[0] cwd = os.getcwd() @@ -38,7 +49,7 @@ class ShowCommand(BaseCommand): module_dir = os.path.join(cwd, module_name) if not module_dir else module_dir if not module_dir or not os.path.exists(module_dir): - return + return True module = Module(module_dir=module_dir) show_text = "Name:%s\n" % module.name @@ -50,6 +61,7 @@ class ShowCommand(BaseCommand): show_text += "Location:%s\n" % module_dir #TODO(wuzewu): add more signature info print(show_text) + return True command = ShowCommand.instance() diff --git a/paddle_hub/commands/uninstall.py b/paddle_hub/commands/uninstall.py index b31bd35c83d35491b201e1090c6cae1e7e23068f..e06bf713c5d9eccd77551591f7496dc058aae0bb 100644 --- a/paddle_hub/commands/uninstall.py +++ b/paddle_hub/commands/uninstall.py @@ -16,9 +16,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function from paddle_hub.tools.logger import logger -from paddle_hub.commands.base_command import BaseCommand +from paddle_hub.commands.base_command import BaseCommand, ENTRY from paddle_hub.tools import utils from paddle_hub.module.manager import default_module_manager +import argparse class UninstallCommand(BaseCommand): @@ -28,13 +29,22 @@ class UninstallCommand(BaseCommand): super(UninstallCommand, self).__init__(name) self.show_in_help = True self.description = "Uninstall the specify module from current environment." + self.parser = self.parser = argparse.ArgumentParser( + description=self.__class__.__doc__, + prog='%s %s ' % (ENTRY, name), + usage='%(prog)s', + add_help=False) def exec(self, argv): + if not argv: + print("ERROR: Please specify a module\n") + self.help() + return False module_name = argv[0] result, tips = default_module_manager.uninstall_module( module_name=module_name) print(tips) - return result + return True command = UninstallCommand.instance() diff --git a/paddle_hub/commands/version.py b/paddle_hub/commands/version.py index 70f6312673371dd40e846b321d37c9a399f91096..ee1fb28204458431b13e534dc7be519adb967d18 100644 --- a/paddle_hub/commands/version.py +++ b/paddle_hub/commands/version.py @@ -30,6 +30,7 @@ class VersionCommand(BaseCommand): def exec(self, argv): print("hub %s" % version.hub_version) + return True command = VersionCommand.instance()