Fix module compatibility issues

paddlehub/__init__.py

@@ -15,10 +15,13 @@
 
 import sys
 
+from easydict import EasyDict
+
 __version__ = '2.0.0a0'
 
 from paddlehub.utils import log, parser, utils
 from paddlehub.module import Module
+
 # In order to maintain the compatibility of the old version, we put the relevant
 # compatible code in the paddlehub/compat package, and mapped some modules referenced
 # in the old version
@@ -26,8 +29,12 @@ from paddlehub.compat import paddle_utils
 from paddlehub.compat.module.processor import BaseProcessor
 from paddlehub.compat.module.nlp_module import NLPPredictionModule, TransformerModule
 from paddlehub.compat.type import DataType
+from paddlehub.compat import task
 
 sys.modules['paddlehub.io.parser'] = parser
 sys.modules['paddlehub.common.logger'] = log
 sys.modules['paddlehub.common.paddle_helper'] = paddle_utils
 sys.modules['paddlehub.common.utils'] = utils
+sys.modules['paddlehub.reader'] = task
+
+common = EasyDict(paddle_helper=paddle_utils)

paddlehub/compat/module/module_v1.py

@@ -177,16 +177,21 @@ class ModuleV1(object):
         return []
 
     @classmethod
-    def load(cls, desc_file):
-        desc = module_v1_utils.convert_module_desc(desc_file)
-
-        cls.author = desc.module_info.author
-        cls.author_email = desc.module_info.author_email
-        cls.summary = desc.module_info.summary
-        cls.type = desc.module_info.type
-        cls.name = desc.module_info.name
-        cls.version = utils.Version(desc.module_info.version)
+    def load(cls, directory: str) -> EasyDict:
+        module_info = cls.load_module_info(directory)
+        cls.name = module_info.name
+        cls.author = module_info.author
+        cls.author_email = module_info.author_email
+        cls.type = module_info.type
+        cls.summary = module_info.summary
+        cls.version = utils.Version(module_info.version)
         return cls
 
+    @classmethod
+    def load_module_info(cls, directory: str) -> EasyDict:
+        desc_file = os.path.join(directory, 'module_desc.pb')
+        desc = module_v1_utils.convert_module_desc(desc_file)
+        return desc.module_info
+
     def assets_path(self):
         return os.path.join(self.directory, 'assets')

paddlehub/compat/module/module_v1_utils.py

@@ -31,13 +31,11 @@ def convert_module_desc(desc_file):
 def convert_signatures(signmaps):
     _dict = EasyDict()
     for sign, var in signmaps.items():
-        _dict[sign] = EasyDict()
+        _dict[sign] = EasyDict(inputs=[], outputs=[])
         for fetch_var in var.fetch_desc:
-            _dict[sign].outputs = list()
             _dict[sign].outputs.append(EasyDict(name=fetch_var.var_name, alias=fetch_var.alias))
 
         for feed_var in var.feed_desc:
-            _dict[sign].inputs = list()
             _dict[sign].inputs.append(EasyDict(name=feed_var.var_name, alias=feed_var.alias))
 
     return _dict

paddlehub/compat/module/nlp_module.py

@@ -22,7 +22,6 @@ from typing import Any, List, Text, Tuple
 
 import paddle
 import numpy as np
-from paddle.fluid.core import PaddleTensor, AnalysisConfig, create_paddle_predictor
 
 from paddlehub.compat import paddle_utils
 from paddlehub.compat.task.transformer_emb_task import TransformerEmbeddingTask
@@ -51,10 +50,10 @@ class NLPBaseModule(RunModule):
 class NLPPredictionModule(NLPBaseModule):
     def _set_config(self):
         '''predictor config setting'''
-        cpu_config = AnalysisConfig(self.pretrained_model_path)
+        cpu_config = paddle.device.core.AnalysisConfig(self.pretrained_model_path)
         cpu_config.disable_glog_info()
         cpu_config.disable_gpu()
-        self.cpu_predictor = create_paddle_predictor(cpu_config)
+        self.cpu_predictor = paddle.device.core.create_paddle_predictor(cpu_config)
 
         try:
             _places = os.environ['CUDA_VISIBLE_DEVICES']
@@ -63,10 +62,10 @@ class NLPPredictionModule(NLPBaseModule):
         except:
             use_gpu = False
         if use_gpu:
-            gpu_config = AnalysisConfig(self.pretrained_model_path)
+            gpu_config = paddle.device.core.AnalysisConfig(self.pretrained_model_path)
             gpu_config.disable_glog_info()
             gpu_config.enable_use_gpu(memory_pool_init_size_mb=500, device_id=0)
-            self.gpu_predictor = create_paddle_predictor(gpu_config)
+            self.gpu_predictor = paddle.device.core.create_paddle_predictor(gpu_config)
 
     def texts2tensor(self, texts: List[dict]) -> paddle.Tensor:
         '''
@@ -82,7 +81,7 @@ class NLPPredictionModule(NLPBaseModule):
         for i, text in enumerate(texts):
             data += text['processed']
             lod.append(len(text['processed']) + lod[i])
-        tensor = PaddleTensor(np.array(data).astype('int64'))
+        tensor = paddle.device.core.PaddleTensor(np.array(data).astype('int64'))
         tensor.name = 'words'
         tensor.lod = [lod]
         tensor.shape = [lod[-1], 1]
@@ -183,7 +182,7 @@ class TransformerModule(NLPBaseModule):
         assert os.path.exists(pretraining_params_path), '[{}] cann\'t be found.'.format(pretraining_params_path)
 
         def existed_params(var):
-            if not isinstance(var, paddle.fluid.framework.Parameter):
+            if not isinstance(var, paddle.device.framework.Parameter):
                 return False
             return os.path.exists(os.path.join(pretraining_params_path, var.name))
 

paddlehub/module/manager.py

@@ -15,6 +15,7 @@
 
 import os
 import shutil
+import sys
 from collections import OrderedDict
 from typing import List
 
@@ -65,11 +66,18 @@ class LocalModuleManager(object):
         self.home = home
         self._local_modules = OrderedDict()
 
+        # Most HubModule can be regarded as a python package, so we need to add the home
+        # directory to sys.path
+        if not home in sys.path:
+            sys.path.insert(0, home)
+
     def _get_normalized_path(self, name: str) -> str:
+        return os.path.join(self.home, self._get_normalized_name(name))
+
+    def _get_normalized_name(self, name: str) -> str:
         # Some HubModules contain '-'  in name (eg roberta_wwm_ext_chinese_L-3_H-1024_A-16).
         # Replace '-' with '_' to comply with python naming conventions.
-        name = name.replace('-', '_')
-        return os.path.join(self.home, name)
+        return name.replace('-', '_')
 
     def install(self,
                 name: str = None,
@@ -194,25 +202,41 @@ class LocalModuleManager(object):
 
     def _install_from_directory(self, directory: str) -> HubModule:
         '''Install a HubModule from directory containing module.py'''
-        hub_module_cls = HubModule.load(directory)
-
-        # Uninstall local module
-        if self.search(hub_module_cls.name):
-            self.uninstall(hub_module_cls.name)
-
-        shutil.copytree(directory, os.path.join(self.home, hub_module_cls.name))
-        self._local_modules[hub_module_cls.name] = hub_module_cls
-
-        for py_req in hub_module_cls.get_py_requirements():
-            log.logger.info('Installing dependent packages: {}'.format(py_req))
-            result = pypi.install(py_req)
-            if result:
-                log.logger.info('Successfully installed {}'.format(py_req))
-            else:
-                log.logger.info('Some errors occurred while installing {}'.format(py_req))
+        module_info = HubModule.load_module_info(directory)
+
+        # A temporary directory is copied here for two purposes:
+        # 1. Avoid affecting user-specified directory (for example, a __pycache__
+        #    directory will be generated).
+        # 2. HubModule is essentially a python package. When internal package
+        #    references are made in it, the correct package name is required.
+        with utils.generate_tempdir() as _dir:
+            tempdir = os.path.join(_dir, module_info.name)
+            tempdir = self._get_normalized_name(tempdir)
+            shutil.copytree(directory, tempdir)
+
+            directory = tempdir
+            hub_module_cls = HubModule.load(directory)
+
+            # Uninstall local module
+            if self.search(hub_module_cls.name):
+                self.uninstall(hub_module_cls.name)
+
+            shutil.copytree(directory, self._get_normalized_path(hub_module_cls.name))
+
+            # Reload the Module object to avoid path errors
+            hub_module_cls = HubModule.load(self._get_normalized_path(hub_module_cls.name))
+            self._local_modules[hub_module_cls.name] = hub_module_cls
+
+            for py_req in hub_module_cls.get_py_requirements():
+                log.logger.info('Installing dependent packages: {}'.format(py_req))
+                result = pypi.install(py_req)
+                if result:
+                    log.logger.info('Successfully installed {}'.format(py_req))
+                else:
+                    log.logger.info('Some errors occurred while installing {}'.format(py_req))
 
-        log.logger.info('Successfully installed {}-{}'.format(hub_module_cls.name, hub_module_cls.version))
-        return hub_module_cls
+            log.logger.info('Successfully installed {}-{}'.format(hub_module_cls.name, hub_module_cls.version))
+            return hub_module_cls
 
     def _install_from_archive(self, archive: str) -> HubModule:
         '''Install HubModule from archive file (eg xxx.tar.gz)'''

paddlehub/module/module.py

@@ -13,12 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import ast
 import inspect
 import importlib
 import os
 import sys
 from typing import Callable, Generic, List, Optional
 
+from easydict import EasyDict
+
 from paddlehub.utils import log, utils
 from paddlehub.compat.module.module_v1 import ModuleV1
 
@@ -81,7 +84,7 @@ class Module(object):
         # If module description file existed, try to load as ModuleV1
         desc_file = os.path.join(directory, 'module_desc.pb')
         if os.path.exists(desc_file):
-            return ModuleV1.load(desc_file)
+            return ModuleV1.load(directory)
 
         basename = os.path.split(directory)[-1]
         dirname = os.path.join(*list(os.path.split(directory)[:-1]))
@@ -98,6 +101,32 @@ class Module(object):
         user_module_cls.directory = directory
         return user_module_cls
 
+    @classmethod
+    def load_module_info(cls, directory: str) -> EasyDict:
+        # If is ModuleV1
+        desc_file = os.path.join(directory, 'module_desc.pb')
+        if os.path.exists(desc_file):
+            return ModuleV1.load_module_info(directory)
+
+        # If is ModuleV2
+        module_file = os.path.join(directory, 'module.py')
+        with open(module_file, 'r') as file:
+            pycode = file.read()
+            ast_module = ast.parse(pycode)
+
+            for _body in ast_module.body:
+                if not isinstance(_body, ast.ClassDef):
+                    continue
+
+                for _decorator in _body.decorator_list:
+                    if _decorator.func.id != 'moduleinfo':
+                        continue
+
+                    info = {key.arg: key.value.s for key in _decorator.keywords}
+                    return EasyDict(info)
+            else:
+                raise InvalidHubModule(directory)
+
     @classmethod
     def init_with_name(cls, name: str, version: str = None, **kwargs):
         '''
@@ -108,7 +137,7 @@ class Module(object):
         if not user_module_cls or not user_module_cls.version.match(version):
             user_module_cls = manager.install(name, version)
 
-        directory = manager._get_normalized_path(name)
+        directory = manager._get_normalized_path(user_module_cls.name)
 
         # The HubModule in the old version will use the _initialize method to initialize,
         # this function will be obsolete in a future version