update module check strategy

paddlehub/common/utils.py

@@ -29,6 +29,27 @@ from paddlehub.module import module_desc_pb2
 from paddlehub.common.logger import logger
 
 
+def version_compare(version1, version2):
+    version1 = version1.split(".")
+    version2 = version2.split(".")
+    num = min(len(version1), len(version2))
+    for index in range(num):
+        try:
+            vn1 = int(version1[index])
+        except:
+            vn1 = 0
+        try:
+            vn2 = int(version2[index])
+        except:
+            vn2 = 0
+
+        if vn1 > vn2:
+            return True
+        elif vn1 < vn2:
+            return False
+    return len(version1) > len(version2)
+
+
 def get_platform():
     return platform.platform()
 
@@ -205,10 +226,3 @@ def get_running_device_info(config):
         dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
 
     return place, dev_count
-
-
-if __name__ == "__main__":
-    print(is_yaml_file("test.yml"))
-    print(is_csv_file("test.yml"))
-    print(is_yaml_file("test.csv"))
-    print(is_csv_file("test.csv"))

paddlehub/module/checker.py

@@ -21,6 +21,7 @@ import os
 import paddle
 
 from paddlehub.common.logger import logger
+from paddlehub.common.utils import version_compare
 from paddlehub.module import check_info_pb2
 from paddlehub.version import hub_version, module_proto_version
 
@@ -81,14 +82,15 @@ class ModuleChecker(object):
         return self.check_info.file_infos
 
     def check(self):
+        result = True
         self.check_info_pb_path = os.path.join(self.module_path,
                                                CHECK_INFO_PB_FILENAME)
 
         if not (os.path.exists(self.check_info_pb_path)
                 or os.path.isfile(self.check_info_pb_path)):
-            logger.error("this module lack of key documents [%s]" %
-                         CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "This module lacks core file %s" % CHECK_INFO_PB_FILENAME)
+            result = False
 
         self.check_info = check_info_pb2.CheckInfo()
         try:
@@ -97,37 +99,52 @@ class ModuleChecker(object):
                 result = self.check_info.ParseFromString(pb_string)
                 if len(pb_string) == 0 or (result is not None
                                            and result != len(pb_string)):
-                    logger.error(
-                        "the [%s] file is incomplete" % CHECK_INFO_PB_FILENAME)
-                    return False
+                    logger.warning(
+                        "File [%s] is incomplete" % CHECK_INFO_PB_FILENAME)
+                    result = False
         except Exception as e:
-            return False
+            result = False
 
         if not self.check_info.paddle_version:
-            logger.error(
-                "can't read paddle version from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning("Unable to read paddle version from [%s]" %
+                           CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.hub_version:
-            logger.error(
-                "can't read hub version from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "Unable to read hub version from [%s]" % CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.module_proto_version:
-            logger.error("can't read module pb version from [%s]" %
-                         CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning("Unable to read module pb version from [%s]" %
+                           CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.file_infos:
-            logger.error(
-                "can't read file info from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "Unable to read file info from [%s]" % CHECK_INFO_PB_FILENAME)
+            result = False
+
+        if not self.check_module():
+            result = False
+
+        if not self.check_compatibility():
+            result = False
 
-        return self.check_module() and self.check_compatibility()
+        return result
 
     def check_compatibility(self):
-        return self._check_module_proto_version() and self._check_hub_version(
-        ) and self._check_paddle_version()
+        result = True
+        if not self._check_module_proto_version():
+            result = False
+
+        if not self._check_hub_version():
+            result = False
+
+        if not self._check_paddle_version():
+            result = False
+
+        return result
 
     def check_module(self):
         return self._check_module_integrity() and self._check_dependency()
@@ -137,34 +154,48 @@ class ModuleChecker(object):
 
     def _check_module_proto_version(self):
         if self.module_proto_version != module_proto_version:
+            logger.warning(
+                "Module description file version cannot be aligned with paddlehub version"
+            )
             return False
         return True
 
     def _check_hub_version(self):
+        if version_compare(self.hub_version, hub_version):
+            logger.warning(
+                "This Module is generated by the paddlehub with version %s, and the local paddlehub version is %s, which may cause serious incompatible bug. Please upgrade paddlehub to the latest version."
+                % (self.hub_version, hub_version))
+            return False
         return True
 
     def _check_paddle_version(self):
+        if version_compare(self.paddle_version, paddle.__version__):
+            logger.warning(
+                "This Module is generated by the paddlepaddle with version %s, and the local paddlepaddle version is %s, which may cause serious incompatible bug. Please upgrade paddlepaddle to the latest version."
+                % (self.paddle_version, paddle.__version__))
+            return False
         return True
 
     def _check_module_integrity(self):
-
+        result = True
         for file_info in self.file_infos:
             file_type = file_info.type
             file_path = file_info.file_name.replace(FILE_SEP, os.sep)
             file_path = os.path.join(self.module_path, file_path)
             if not os.path.exists(file_path):
                 if file_info.is_need:
-                    logger.error(
-                        "Module incompleted! Missing file [%s]" % file_path)
-                    return False
+                    logger.warning(
+                        "Module integrity check failed! Missing file [%s]" %
+                        file_path)
+                    result = False
             else:
                 if file_type == check_info_pb2.FILE:
                     if not os.path.isfile(file_path):
-                        logger.error("File type error %s" % file_path)
-                        return False
+                        logger.warning("File type check error %s" % file_path)
+                        result = False
 
                 if file_type == check_info_pb2.DIR:
                     if not os.path.isdir(file_path):
-                        logger.error("File type error %s" % file_path)
-                        return False
-        return True
+                        logger.warning("File type check error %s" % file_path)
+                        result = False
+        return result

paddlehub/module/module.py

@@ -191,10 +191,7 @@ class Module(object):
 
     def _init_with_module_file(self, module_dir):
         checker = ModuleChecker(module_dir)
-        if not checker.check():
-            logger.error(
-                "Module initialization failed on {}".format(module_dir))
-            exit(1)
+        checker.check()
 
         self.helper = ModuleHelper(module_dir)
         with open(self.helper.module_desc_path(), "rb") as fi: