Merge branch 'develop' into update_v2_0301

paddlehub/commands/install.py

@@ -46,7 +46,7 @@ class InstallCommand(BaseCommand):
             return False
         extra = {"command": "install"}
 
-        if argv[0].endswith("tar.gz") or argv[0].endswith("phm"):
+        if argv[0].endswith("tar.gz"):
             result, tips, module_dir = default_module_manager.install_module(
                 module_package=argv[0], extra=extra)
         elif os.path.exists(argv[0]) and os.path.isdir(argv[0]):

paddlehub/module/check_info.proto

@@ -52,5 +52,4 @@ message CheckInfo {
   string module_proto_version = 3;
   repeated FileInfo file_infos = 4;
   repeated Requires requires = 5;
-  string module_code_version = 6;
 };

paddlehub/module/check_info_pb2.py

@@ -18,7 +18,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
     package='paddlehub.module.checkinfo',
     syntax='proto3',
     serialized_pb=_b(
-        '\n\x10\x63heck_info.proto\x12\x1apaddlehub.module.checkinfo\"\x85\x01\n\x08\x46ileInfo\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x33\n\x04type\x18\x02 \x01(\x0e\x32%.paddlehub.module.checkinfo.FILE_TYPE\x12\x0f\n\x07is_need\x18\x03 \x01(\x08\x12\x0b\n\x03md5\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\"\x84\x01\n\x08Requires\x12>\n\x0crequire_type\x18\x01 \x01(\x0e\x32(.paddlehub.module.checkinfo.REQUIRE_TYPE\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\ngreat_than\x18\x03 \x01(\x08\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\"\xe5\x01\n\tCheckInfo\x12\x16\n\x0epaddle_version\x18\x01 \x01(\t\x12\x13\n\x0bhub_version\x18\x02 \x01(\t\x12\x1c\n\x14module_proto_version\x18\x03 \x01(\t\x12\x38\n\nfile_infos\x18\x04 \x03(\x0b\x32$.paddlehub.module.checkinfo.FileInfo\x12\x36\n\x08requires\x18\x05 \x03(\x0b\x32$.paddlehub.module.checkinfo.Requires\x12\x1b\n\x13module_code_version\x18\x06 \x01(\t*\x1e\n\tFILE_TYPE\x12\x08\n\x04\x46ILE\x10\x00\x12\x07\n\x03\x44IR\x10\x01*[\n\x0cREQUIRE_TYPE\x12\x12\n\x0ePYTHON_PACKAGE\x10\x00\x12\x0e\n\nHUB_MODULE\x10\x01\x12\n\n\x06SYSTEM\x10\x02\x12\x0b\n\x07\x43OMMAND\x10\x03\x12\x0e\n\nPY_VERSION\x10\x04\x42\x02H\x03\x62\x06proto3'
+        '\n\x10\x63heck_info.proto\x12\x1apaddlehub.module.checkinfo\"\x85\x01\n\x08\x46ileInfo\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x33\n\x04type\x18\x02 \x01(\x0e\x32%.paddlehub.module.checkinfo.FILE_TYPE\x12\x0f\n\x07is_need\x18\x03 \x01(\x08\x12\x0b\n\x03md5\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\"\x84\x01\n\x08Requires\x12>\n\x0crequire_type\x18\x01 \x01(\x0e\x32(.paddlehub.module.checkinfo.REQUIRE_TYPE\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\ngreat_than\x18\x03 \x01(\x08\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\"\xc8\x01\n\tCheckInfo\x12\x16\n\x0epaddle_version\x18\x01 \x01(\t\x12\x13\n\x0bhub_version\x18\x02 \x01(\t\x12\x1c\n\x14module_proto_version\x18\x03 \x01(\t\x12\x38\n\nfile_infos\x18\x04 \x03(\x0b\x32$.paddlehub.module.checkinfo.FileInfo\x12\x36\n\x08requires\x18\x05 \x03(\x0b\x32$.paddlehub.module.checkinfo.Requires*\x1e\n\tFILE_TYPE\x12\x08\n\x04\x46ILE\x10\x00\x12\x07\n\x03\x44IR\x10\x01*[\n\x0cREQUIRE_TYPE\x12\x12\n\x0ePYTHON_PACKAGE\x10\x00\x12\x0e\n\nHUB_MODULE\x10\x01\x12\n\n\x06SYSTEM\x10\x02\x12\x0b\n\x07\x43OMMAND\x10\x03\x12\x0e\n\nPY_VERSION\x10\x04\x42\x02H\x03\x62\x06proto3'
     ))
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
@@ -35,8 +35,8 @@ _FILE_TYPE = _descriptor.EnumDescriptor(
     ],
     containing_type=None,
     options=None,
-    serialized_start=551,
-    serialized_end=581,
+    serialized_start=522,
+    serialized_end=552,
 )
 _sym_db.RegisterEnumDescriptor(_FILE_TYPE)
 
@@ -60,8 +60,8 @@ _REQUIRE_TYPE = _descriptor.EnumDescriptor(
     ],
     containing_type=None,
     options=None,
-    serialized_start=583,
-    serialized_end=674,
+    serialized_start=554,
+    serialized_end=645,
 )
 _sym_db.RegisterEnumDescriptor(_REQUIRE_TYPE)
 
@@ -346,22 +346,6 @@ _CHECKINFO = _descriptor.Descriptor(
             is_extension=False,
             extension_scope=None,
             options=None),
-        _descriptor.FieldDescriptor(
-            name='module_code_version',
-            full_name='paddlehub.module.checkinfo.CheckInfo.module_code_version',
-            index=5,
-            number=6,
-            type=9,
-            cpp_type=9,
-            label=1,
-            has_default_value=False,
-            default_value=_b("").decode('utf-8'),
-            message_type=None,
-            enum_type=None,
-            containing_type=None,
-            is_extension=False,
-            extension_scope=None,
-            options=None),
     ],
     extensions=[],
     nested_types=[],
@@ -372,7 +356,7 @@ _CHECKINFO = _descriptor.Descriptor(
     extension_ranges=[],
     oneofs=[],
     serialized_start=320,
-    serialized_end=549,
+    serialized_end=520,
 )
 
 _FILEINFO.fields_by_name['type'].enum_type = _FILE_TYPE

paddlehub/module/manager.py

@@ -22,6 +22,9 @@ import shutil
 
 from functools import cmp_to_key
 import tarfile
+import sys
+import importlib
+import inspect
 
 import paddlehub as hub
 from paddlehub.common import utils
@@ -54,6 +57,22 @@ class LocalModuleManager(object):
                     "version"].s
                 return True, info
             else:
+                module_file = os.path.join(module_path, 'module.py')
+                if os.path.exists(module_file):
+                    basename = os.path.split(module_path)[-1]
+                    dirname = os.path.join(
+                        *list(os.path.split(module_path)[:-1]))
+                    sys.path.insert(0, dirname)
+                    _module = importlib.import_module(
+                        "{}.module".format(basename))
+                    for _item, _cls in inspect.getmembers(
+                            _module, inspect.isclass):
+                        _item = _module.__dict__[_item]
+                        if issubclass(_item, hub.Module):
+                            version = _item._version
+                            break
+                    sys.path.pop(0)
+                    return True, {'version': version}
                 logger.warning(
                     "%s does not exist, the module will be reinstalled" %
                     desc_pb_path)

paddlehub/module/module.py

@@ -54,72 +54,6 @@ PYTHON_DIR = "python"
 PROCESSOR_NAME = "processor"
 # PaddleHub var prefix
 HUB_VAR_PREFIX = "@HUB_%s@"
-# PaddleHub Module package suffix
-HUB_PACKAGE_SUFFIX = "phm"
-
-
-def create_module(directory, name, author, email, module_type, summary,
-                  version):
-    save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX)
-
-    with tmp_dir() as base_dir:
-        # package the module
-        with tarfile.open(save_file, "w:gz") as tar:
-            module_dir = os.path.join(base_dir, name)
-            shutil.copytree(directory, module_dir)
-
-            # record module info and serialize
-            desc = module_desc_pb2.ModuleDesc()
-            attr = desc.attr
-            attr.type = module_desc_pb2.MAP
-            module_info = attr.map.data['module_info']
-            module_info.type = module_desc_pb2.MAP
-            utils.from_pyobj_to_module_attr(name, module_info.map.data['name'])
-            utils.from_pyobj_to_module_attr(author,
-                                            module_info.map.data['author'])
-            utils.from_pyobj_to_module_attr(
-                email, module_info.map.data['author_email'])
-            utils.from_pyobj_to_module_attr(module_type,
-                                            module_info.map.data['type'])
-            utils.from_pyobj_to_module_attr(summary,
-                                            module_info.map.data['summary'])
-            utils.from_pyobj_to_module_attr(version,
-                                            module_info.map.data['version'])
-            module_desc_path = os.path.join(module_dir, "module_desc.pb")
-            with open(module_desc_path, "wb") as f:
-                f.write(desc.SerializeToString())
-
-            # generate check info
-            checker = ModuleChecker(module_dir)
-            checker.generate_check_info()
-
-            # add __init__
-            module_init = os.path.join(module_dir, "__init__.py")
-            with open(module_init, "a") as file:
-                file.write("")
-
-            _cwd = os.getcwd()
-            os.chdir(base_dir)
-            module_dir = module_dir.replace(base_dir, ".")
-            tar.add(module_dir, recursive=False)
-            files = []
-            for dirname, _, subfiles in os.walk(module_dir):
-                for file in subfiles:
-                    #                     if file.startswith("."):
-                    #                         continue
-                    files.append(os.path.join(dirname, file))
-
-            total_length = len(files)
-            print("Create Module {}-{}".format(name, version))
-            for index, file in enumerate(files):
-                done = int(float(index) / total_length * 50)
-                progress("[%-50s] %.2f%%" % ('=' * done,
-                                             float(index / total_length * 100)))
-                tar.add(file)
-            progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True)
-            print("Module package saved as {}".format(save_file))
-            os.chdir(_cwd)
-
 
 _module_runnable_func = {}
 
@@ -147,10 +81,22 @@ def serving(func):
     return _wrapper
 
 
-class Module(object):
+def moduleinfo(name, version, author, author_email, summary, type):
+    def _wrapper(cls):
+        if not issubclass(cls, Module):
+            raise RuntimeError
+        cls._name = name
+        cls._version = version
+        cls._author = author
+        cls._author_email = author_email
+        cls._summary = summary
+        cls._type = type
+        return cls
+
+    return _wrapper
 
-    _record = {}
 
+class Module(object):
     def __new__(cls,
                 name=None,
                 directory=None,
@@ -176,7 +122,13 @@ class Module(object):
                 module = cls.init_with_directory(directory=directory, **kwargs)
             CacheUpdater("update_cache", module.name, module.version).start()
         else:
-            module = object.__new__(cls)
+            if not name and not directory:
+                directory = os.path.dirname(
+                    os.path.abspath(sys.modules[cls.__module__].__file__))
+                module = Module.init_with_directory(
+                    directory=directory, **kwargs)
+            else:
+                module = object.__new__(cls)
 
         return module
 
@@ -187,9 +139,8 @@ class Module(object):
                  version=None,
                  **kwargs):
         # Avoid module being initialized multiple times
-        if not directory or id(self) in Module._record:
+        if "_is_initialize" in self.__dict__ and self._is_initialize:
             return
-        Module._record[id(self)] = True
 
         mod = self.__class__.__module__ + "." + self.__class__.__name__
         if mod in _module_runnable_func:
@@ -200,26 +151,8 @@ class Module(object):
         self._serving_func_name = _module_serving_func.get(mod, None)
         self._code_version = "v2"
         self._directory = directory
-        self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME)
-        self._desc = module_desc_pb2.ModuleDesc()
-        with open(self.module_desc_path, "rb") as file:
-            self._desc.ParseFromString(file.read())
-
-        module_info = self.desc.attr.map.data['module_info']
-        self._name = utils.from_module_attr_to_pyobj(
-            module_info.map.data['name'])
-        self._author = utils.from_module_attr_to_pyobj(
-            module_info.map.data['author'])
-        self._author_email = utils.from_module_attr_to_pyobj(
-            module_info.map.data['author_email'])
-        self._version = utils.from_module_attr_to_pyobj(
-            module_info.map.data['version'])
-        self._type = utils.from_module_attr_to_pyobj(
-            module_info.map.data['type'])
-        self._summary = utils.from_module_attr_to_pyobj(
-            module_info.map.data['summary'])
-
         self._initialize(**kwargs)
+        self._is_initialize = True
 
     @classmethod
     def init_with_name(cls, name, version=None, **kwargs):
@@ -243,64 +176,54 @@ class Module(object):
     @classmethod
     def init_with_directory(cls, directory, **kwargs):
         desc_file = os.path.join(directory, MODULE_DESC_PBNAME)
-        checker = ModuleChecker(directory)
-        checker.check()
-
-        module_code_version = checker.module_code_version
-        if module_code_version == "v2":
-            sys.path.insert(0, directory)
-            # clear module cache
-            if 'module' in sys.modules:
-                sys.modules.pop('module')
-            _module = importlib.import_module("module")
-            for _item, _cls in inspect.getmembers(_module, inspect.isclass):
-                _item = _module.__dict__[_item]
-                if issubclass(_item, Module):
-                    user_module = _item(directory=directory, **kwargs)
-                    break
-            sys.path.pop(0)
-            return user_module
-        return ModuleV1(directory=directory, **kwargs)
+        if os.path.exists(desc_file):
+            checker = ModuleChecker(directory)
+            checker.check()
+            return ModuleV1(directory=directory, **kwargs)
+
+        basename = os.path.split(directory)[-1]
+        dirname = os.path.join(*list(os.path.split(directory)[:-1]))
+        sys.path.insert(0, dirname)
+        _module = importlib.import_module("{}.module".format(basename))
+        for _item, _cls in inspect.getmembers(_module, inspect.isclass):
+            _item = _module.__dict__[_item]
+            if issubclass(_item, Module):
+                user_module = _item(directory=directory, **kwargs)
+                break
+        sys.path.pop(0)
+        return user_module
 
     @property
     def run_func(self):
         return self._run_func
 
-    @property
-    def desc(self):
-        return self._desc
-
     @property
     def directory(self):
         return self._directory
 
     @property
     def author(self):
-        return self._author
+        return self.__class__._author
 
     @property
     def author_email(self):
-        return self._author_email
+        return self.__class__._author_email
 
     @property
     def summary(self):
-        return self._summary
+        return self.__class__._summary
 
     @property
     def type(self):
-        return self._type
+        return self.__class__._type
 
     @property
     def version(self):
-        return self._version
+        return self.__class__._version
 
     @property
     def name(self):
-        return self._name
-
-    @property
-    def code_version(self):
-        return self._code_version
+        return self.__class__._name
 
     @property
     def is_runnable(self):
@@ -340,7 +263,6 @@ class ModuleV1(Module):
         if not directory:
             return
         super(ModuleV1, self).__init__(name, directory, module_dir, version)
-        self._code_version = "v1"
         self.program = None
         self.assets = []
         self.helper = None
@@ -349,6 +271,26 @@ class ModuleV1(Module):
         self.processor = None
         self.extra_info = {}
 
+        # parse desc
+        self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME)
+        self._desc = module_desc_pb2.ModuleDesc()
+        with open(self.module_desc_path, "rb") as file:
+            self._desc.ParseFromString(file.read())
+
+        module_info = self.desc.attr.map.data['module_info']
+        self._name = utils.from_module_attr_to_pyobj(
+            module_info.map.data['name'])
+        self._author = utils.from_module_attr_to_pyobj(
+            module_info.map.data['author'])
+        self._author_email = utils.from_module_attr_to_pyobj(
+            module_info.map.data['author_email'])
+        self._version = utils.from_module_attr_to_pyobj(
+            module_info.map.data['version'])
+        self._type = utils.from_module_attr_to_pyobj(
+            module_info.map.data['type'])
+        self._summary = utils.from_module_attr_to_pyobj(
+            module_info.map.data['summary'])
+
         # cache data
         self.last_call_name = None
         self.cache_feed_dict = None
@@ -376,6 +318,33 @@ class ModuleV1(Module):
         serving_func_name = self.desc.attr.map.data['default_signature'].s
         return serving_func_name if serving_func_name != "" else None
 
+    def desc(self):
+        return self._desc
+
+    @property
+    def author(self):
+        return self._author
+
+    @property
+    def author_email(self):
+        return self._author_email
+
+    @property
+    def summary(self):
+        return self._summary
+
+    @property
+    def type(self):
+        return self._type
+
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def name(self):
+        return self._name
+
     def _dump_processor(self):
         import inspect
         pymodule = inspect.getmodule(self.processor)