Merge branch 'develop' of https://github.com/PaddlePaddle/PaddleHub into develop

.copyright.hook

+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import io, re
+import sys, os
+import subprocess
+import platform
+
+COPYRIGHT = '''
+  Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+LANG_COMMENT_MARK = None
+
+NEW_LINE_MARK = None
+
+COPYRIGHT_HEADER = None
+
+if platform.system() == "Windows":
+    NEW_LINE_MARK = "\r\n"
+else:
+    NEW_LINE_MARK = '\n'
+    COPYRIGHT_HEADER = COPYRIGHT.split(NEW_LINE_MARK)[1]
+    p = re.search('(\d{4})', COPYRIGHT_HEADER).group(0)
+    process = subprocess.Popen(["date", "+%Y"], stdout=subprocess.PIPE)
+    date, err = process.communicate()
+    date = date.decode("utf-8").rstrip("\n")
+    COPYRIGHT_HEADER = COPYRIGHT_HEADER.replace(p, date)
+
+
+def generate_copyright(template, lang='C'):
+    if lang == 'Python':
+        LANG_COMMENT_MARK = '#'
+    else:
+        LANG_COMMENT_MARK = "//"
+
+    lines = template.split(NEW_LINE_MARK)
+    BLANK = " "
+    ans = LANG_COMMENT_MARK + BLANK + COPYRIGHT_HEADER + NEW_LINE_MARK
+    for lino, line in enumerate(lines):
+        if lino == 0 or lino == 1 or lino == len(lines) - 1: continue
+        if len(line)  == 0:
+            BLANK = ""
+        else:
+            BLANK = " "
+        ans += LANG_COMMENT_MARK + BLANK + line + NEW_LINE_MARK
+
+    return ans + "\n"
+
+
+def lang_type(filename):
+    if filename.endswith(".py"):
+        return "Python"
+    elif filename.endswith(".h"):
+        return "C"
+    elif filename.endswith(".c"):
+        return "C"
+    elif filename.endswith(".hpp"):
+        return "C"
+    elif filename.endswith(".cc"):
+        return "C"
+    elif filename.endswith(".cpp"):
+        return "C"
+    elif filename.endswith(".cu"):
+        return "C"
+    elif filename.endswith(".cuh"):
+        return "C"
+    elif filename.endswith(".go"):
+        return "C"
+    elif filename.endswith(".proto"):
+        return "C"
+    else:
+        print("Unsupported filetype %s", filename)
+        exit(0)
+
+
+PYTHON_ENCODE = re.compile("^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)")
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description='Checker for copyright declaration.')
+    parser.add_argument('filenames', nargs='*', help='Filenames to check')
+    args = parser.parse_args(argv)
+
+    retv = 0
+    for filename in args.filenames:
+        fd = io.open(filename, encoding="utf-8")
+        first_line = fd.readline()
+        second_line = fd.readline()
+        if "COPYRIGHT (C)" in first_line.upper(): continue
+        if first_line.startswith("#!") or PYTHON_ENCODE.match(
+                second_line) != None or PYTHON_ENCODE.match(first_line) != None:
+            continue
+        original_contents = io.open(filename, encoding="utf-8").read()
+        new_contents = generate_copyright(
+            COPYRIGHT, lang_type(filename)) + original_contents
+        print('Auto Insert Copyright Header {}'.format(filename))
+        retv = 1
+        with io.open(filename, 'w') as output_file:
+            output_file.write(new_contents)
+
+    return retv
+
+
+if __name__ == '__main__':
+    exit(main())

paddle_hub/commands/__init__.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import hub
 from . import download
 from . import run
 from . import show
@@ -22,3 +21,4 @@ from . import install
 from . import uninstall
 from . import search
 from . import help
+from . import hub

paddle_hub/commands/base_command.py

@@ -16,9 +16,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.arg_helper import add_argument, print_arguments
-from paddle_hub.tools.logger import logger
 import argparse
 
+ENTRY = "hub"
+
 
 class BaseCommand:
     command_dict = {}
@@ -39,12 +40,14 @@ class BaseCommand:
         assert not hasattr(
             self.__class__,
             '_instance'), 'Please use `instance()` to get Command object!'
-        self.parser = argparse.ArgumentParser(description=__doc__)
         self.args = None
         self.name = name
         self.show_in_help = True
         self.description = ""
 
+    def help(self):
+        self.parser.print_help()
+
     def add_arg(self, argument, type="str", default=None, help=None):
         add_argument(
             argument=argument,

paddle_hub/commands/download.py

@@ -16,10 +16,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.tools import utils
 from paddle_hub.tools.downloader import default_downloader
 from paddle_hub.hub_server import default_hub_server
+import argparse
 
 
 class DownloadCommand(BaseCommand):
@@ -29,15 +30,21 @@ class DownloadCommand(BaseCommand):
         super(DownloadCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Download a paddle hub module."
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <module_name>' % (ENTRY, name),
+            usage='%(prog)s [options]',
+            add_help=False)
         # yapf: disable
-        self.add_arg('--output_path',  str,  ".",   "path to save the module, default in current directory" )
+        self.add_arg('--output_path',  str,  ".",   "path to save the module" )
         self.add_arg('--uncompress',   bool, False,  "uncompress the download package or not" )
         # yapf: enable
 
-    def help(self):
-        self.parser.print_help()
-
     def exec(self, argv):
+        if not argv:
+            print("ERROR: Please specify a module\n")
+            self.help()
+            return False
         module_name = argv[0]
         module_version = None if "==" not in module_name else module_name.split(
             "==")[1]
@@ -55,7 +62,7 @@ class DownloadCommand(BaseCommand):
             if module_version:
                 tips += " with version %s" % module_version
             print(tips)
-            return
+            return True
 
         self.print_args()
         if self.args.uncompress:
@@ -65,7 +72,7 @@ class DownloadCommand(BaseCommand):
             result, tips, file = default_downloader.download_file(
                 url=url, save_path=self.args.output_path)
         print(tips)
-        return result
+        return True
 
 
 command = DownloadCommand.instance()

paddle_hub/commands/help.py

@@ -42,6 +42,7 @@ class HelpCommand(BaseCommand):
             help_text += "  %-15s\t\t%s\n" % (command.name, command.description)
 
         print(help_text)
+        return True
 
 
 command = HelpCommand.instance()

paddle_hub/commands/hub.py

@@ -31,22 +31,24 @@ class HubCommand(BaseCommand):
     def __init__(self, name):
         super(HubCommand, self).__init__(name)
         self.show_in_help = False
-        # yapf: disable
-        self.add_arg('command',  str, None,  "command to run" )
-        # yapf: enable
 
     def exec(self, argv):
-        args = self.parser.parse_args(argv[1:2])
-
-        if not args.command in BaseCommand.command_dict:
-            logger.critical("command %s not supported!" % args.command)
+        if not argv:
+            help.command.exec(argv)
+            exit(1)
+            return False
+        sub_command = argv[0]
+        if not sub_command in BaseCommand.command_dict:
+            print("ERROR: unknown command '%s'" % sub_command)
+            help.command.exec(argv)
             exit(1)
+            return False
 
-        command = BaseCommand.command_dict[args.command]
-        command.exec(argv[2:])
+        command = BaseCommand.command_dict[sub_command]
+        return command.exec(argv[1:])
 
 
 command = HubCommand.instance()
 
 if __name__ == "__main__":
-    command.exec(sys.argv)
+    command.exec(sys.argv[1:])

paddle_hub/commands/install.py

@@ -16,9 +16,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.tools import utils
 from paddle_hub.module.manager import default_module_manager
+import argparse
 
 
 class InstallCommand(BaseCommand):
@@ -28,12 +29,18 @@ class InstallCommand(BaseCommand):
         super(InstallCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Install the specify module to current environment."
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <module_name>' % (ENTRY, name),
+            usage='%(prog)s',
+            add_help=False)
         #TODO(wuzewu): add --upgrade option
 
-    def help(self):
-        self.parser.print_help()
-
     def exec(self, argv):
+        if not argv:
+            print("ERROR: Please specify a module\n")
+            self.help()
+            return False
         module_name = argv[0]
         module_version = None if "==" not in module_name else module_name.split(
             "==")[1]
@@ -42,7 +49,7 @@ class InstallCommand(BaseCommand):
         result, tips, module_dir = default_module_manager.install_module(
             module_name=module_name, module_version=module_version)
         print(tips)
-        return result
+        return True
 
 
 command = InstallCommand.instance()

paddle_hub/commands/list.py

@@ -38,6 +38,7 @@ class ListCommand(BaseCommand):
         for module_name, module_dir in all_modules.items():
             list_text += "  %-20s\t\t%s\n" % (module_name, module_dir)
         print(list_text)
+        return True
 
 
 command = ListCommand.instance()

paddle_hub/commands/run.py

@@ -15,13 +15,18 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
-import paddle_hub as hub
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.io.reader import csv_reader, yaml_reader
+from paddle_hub.module.manager import default_module_manager
 from paddle_hub.tools import utils
 from paddle_hub.tools.arg_helper import add_argument, print_arguments
 
+import paddle_hub as hub
+import argparse
+import os
+
 
 class RunCommand(BaseCommand):
     name = "run"
@@ -30,51 +35,59 @@ class RunCommand(BaseCommand):
         super(RunCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Run the specify module"
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <module>' % (ENTRY, name),
+            usage='%(prog)s [options]')
         # yapf: disable
         self.add_arg('--config',    str, None,  "config file in yaml format" )
         self.add_arg('--dataset',   str, None,  "dataset be used" )
-        self.add_arg('--module',    str, None,  "module to run" )
         self.add_arg('--signature', str, None,  "signature to run" )
         # yapf: enable
 
-    def _check_module(self):
-        if not self.args.module:
-            logger.critical("lack of module")
-            self.help()
-            exit(1)
-
     def _check_dataset(self):
         if not self.args.dataset:
-            logger.critical("lack of dataset file")
+            print("Error! Lack of dataset file")
             self.help()
             exit(1)
         if not utils.is_csv_file(self.args.dataset):
-            logger.critical("dataset file should in csv format")
+            print("Error! Dataset file should in csv format")
             self.help()
             exit(1)
 
     def _check_config(self):
         if not self.args.config:
-            logger.critical("lack of config file")
+            print("Error! Lack of config file")
             self.help()
             exit(1)
         if not utils.is_yaml_file(self.args.config):
-            logger.critical("config file should in yaml format")
+            print("Error! Config file should in yaml format")
             self.help()
             exit(1)
 
-    def help(self):
-        self.parser.print_help()
-
     def exec(self, argv):
-        self.args = self.parser.parse_args(argv)
-        self.print_args()
-
-        self._check_module()
+        if not argv:
+            print("ERROR: Please specify a key\n")
+            self.help()
+            return False
+        module_name = argv[0]
+        self.args = self.parser.parse_args(argv[1:])
         self._check_dataset()
         self._check_config()
 
-        module = hub.Module(module_dir=self.args.module)
+        module_dir = default_module_manager.search_module(module_name)
+        if not module_dir:
+            if os.path.exists(module_name):
+                module_dir = module_name
+            else:
+                print("Install Module %s" % module_name)
+                result, tips, module_dir = default_module_manager.install_module(
+                    module_name)
+                print(tips)
+                if not result:
+                    return False
+
+        module = hub.Module(module_dir=module_dir)
         yaml_config = yaml_reader.read(self.args.config)
 
         if not self.args.signature:

paddle_hub/commands/search.py

@@ -16,9 +16,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.tools import utils
 from paddle_hub.hub_server import default_hub_server
+import argparse
 
 
 class SearchCommand(BaseCommand):
@@ -28,8 +29,18 @@ class SearchCommand(BaseCommand):
         super(SearchCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Search a paddle hub module with keyword."
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <key>' % (ENTRY, name),
+            usage='%(prog)s',
+            add_help=False)
 
     def exec(self, argv):
+        if not argv:
+            print("ERROR: Please specify a key\n")
+            self.help()
+            return False
+
         module_name = argv[0]
         module_list = default_hub_server.search_module(module_name)
         text = "\n"
@@ -38,6 +49,7 @@ class SearchCommand(BaseCommand):
         for module_name, module_version in module_list:
             text += "  %-20s\t\t%s\n" % (module_name, module_version)
         print(text)
+        return True
 
 
 command = SearchCommand.instance()

paddle_hub/commands/show.py

@@ -16,10 +16,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.module.manager import default_module_manager
 from paddle_hub.module.module import Module
 import os
+import argparse
 
 
 class ShowCommand(BaseCommand):
@@ -29,8 +30,18 @@ class ShowCommand(BaseCommand):
         super(ShowCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Show the specify module's info"
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <module_name/module_dir>' % (ENTRY, name),
+            usage='%(prog)s',
+            add_help=False)
 
     def exec(self, argv):
+        if not argv:
+            print("ERROR: Please specify a module\n")
+            self.help()
+            return False
+
         module_name = argv[0]
 
         cwd = os.getcwd()
@@ -38,7 +49,7 @@ class ShowCommand(BaseCommand):
         module_dir = os.path.join(cwd,
                                   module_name) if not module_dir else module_dir
         if not module_dir or not os.path.exists(module_dir):
-            return
+            return True
 
         module = Module(module_dir=module_dir)
         show_text = "Name:%s\n" % module.name
@@ -50,6 +61,7 @@ class ShowCommand(BaseCommand):
         show_text += "Location:%s\n" % module_dir
         #TODO(wuzewu): add more signature info
         print(show_text)
+        return True
 
 
 command = ShowCommand.instance()

paddle_hub/commands/uninstall.py

@@ -16,9 +16,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from paddle_hub.tools.logger import logger
-from paddle_hub.commands.base_command import BaseCommand
+from paddle_hub.commands.base_command import BaseCommand, ENTRY
 from paddle_hub.tools import utils
 from paddle_hub.module.manager import default_module_manager
+import argparse
 
 
 class UninstallCommand(BaseCommand):
@@ -28,13 +29,22 @@ class UninstallCommand(BaseCommand):
         super(UninstallCommand, self).__init__(name)
         self.show_in_help = True
         self.description = "Uninstall the specify module from current environment."
+        self.parser = self.parser = argparse.ArgumentParser(
+            description=self.__class__.__doc__,
+            prog='%s %s <module_name>' % (ENTRY, name),
+            usage='%(prog)s',
+            add_help=False)
 
     def exec(self, argv):
+        if not argv:
+            print("ERROR: Please specify a module\n")
+            self.help()
+            return False
         module_name = argv[0]
         result, tips = default_module_manager.uninstall_module(
             module_name=module_name)
         print(tips)
-        return result
+        return True
 
 
 command = UninstallCommand.instance()

paddle_hub/commands/version.py

@@ -30,6 +30,7 @@ class VersionCommand(BaseCommand):
 
     def exec(self, argv):
         print("hub %s" % version.hub_version)
+        return True
 
 
 command = VersionCommand.instance()