Merge branch 'develop' of https://github.com/PaddlePaddle/PaddleHub into develop

demo/image-classification/create_module.py

@@ -56,7 +56,7 @@ def create_module(args):
         "feature_map", inputs=[image], outputs=[feature_map])
     hub.create_module(
         sign_arr=[sign1, sign2],
-        module_dir="hub_module_" + args.model,
+        module_dir=args.model + ".hub_module",
         module_info="resources/module_info.yml",
         processor=processor.Processor,
         assets=assets)

demo/image-classification/create_module.sh

@@ -16,7 +16,7 @@ do
 done
 
 script_path=$(cd `dirname $0`; pwd)
-module_path=hub_module_${model_name}
+module_path=${model_name}.hub_module
 
 if [ -d $script_path/$module_path ]
 then

demo/image-classification/infer.sh

-python ../../paddlehub/commands/hub.py run hub_module_ResNet50/ --signature classification --config resources/test/test.yml  --dataset resources/test/test.csv
+python ../../paddlehub/commands/hub.py run ResNet50.hub_module/ --signature classification --config resources/test/test.yml  --dataset resources/test/test.csv

demo/image-classification/resources/module_info.yml

-name: Resnet50
+name: resnet_v2_50_imagenet
 type: CV/classification
 author: paddlepaddle
 author_email: paddle-dev@baidu.com

demo/image-classification/retrain.py

@@ -5,11 +5,11 @@ import paddlehub as hub
 
 
 def train():
-    resnet_module = hub.Module(module_dir="hub_module_ResNet50")
+    resnet_module = hub.Module(module_dir="ResNet50.hub_module")
     input_dict, output_dict, program = resnet_module.context(
         sign_name="feature_map", trainable=True)
     dataset = hub.dataset.Flowers()
-    data_reader = hub.ImageClassificationReader(
+    data_reader = hub.reader.ImageClassificationReader(
         image_width=224, image_height=224, dataset=dataset)
     with fluid.program_guard(program):
         label = fluid.layers.data(name="label", dtype="int64", shape=[1])
@@ -24,7 +24,7 @@ def train():
 
         feed_list = [img.name, label.name]
 
-        task = hub.append_mlp_classifier(
+        task = hub.create_img_classification_task(
             feature=feature_map, label=label, num_classes=dataset.num_labels)
         hub.finetune_and_eval(
             task, feed_list=feed_list, data_reader=data_reader, config=config)

demo/lac/create_module.py

@@ -35,7 +35,7 @@ def create_module():
     # assets
     assets = [word_dict_path, label_dict_path, word_rep_dict_path]
 
-    # create a module and save as hub_module_lac
+    # create a module and save as lac.hub_module
     sign = hub.create_signature(
         name="lexical_analysis",
         inputs=[word],
@@ -43,7 +43,7 @@ def create_module():
         for_predict=True)
     hub.create_module(
         sign_arr=[sign],
-        module_dir="hub_module_lac",
+        module_dir="lac.hub_module",
         exe=exe,
         module_info="resources/module_info.yml",
         processor=processor.Processor,

demo/lac/create_module.sh

@@ -3,7 +3,7 @@ set -o nounset
 set -o errexit
 
 script_path=$(cd `dirname $0`; pwd)
-module_path=hub_module_lac
+module_path=lac.hub_module
 
 if [ -d $script_path/$module_path ]
 then

demo/lac/infer.sh

-python ../../paddlehub/commands/hub.py run hub_module_lac/ --signature lexical_analysis --config resources/test/test.yml  --input_file resources/test/test.txt
+python ../../paddlehub/commands/hub.py run lac.hub_module/ --signature lexical_analysis --config resources/test/test.yml  --input_file resources/test/test.txt

demo/lac/infer_by_code.py

@@ -4,7 +4,7 @@ import paddlehub as hub
 
 def infer_with_input_text():
     # get lac module
-    lac = hub.Module(module_dir="hub_module_lac")
+    lac = hub.Module(module_dir="lac.hub_module")
 
     test_text = ["今天是个好日子", "天气预报说今天要下雨", "下一班地铁马上就要到了"]
 
@@ -24,7 +24,7 @@ def infer_with_input_text():
 
 def infer_with_input_file():
     # get lac module
-    lac = hub.Module(module_dir="hub_module_lac")
+    lac = hub.Module(module_dir="lac.hub_module")
 
     # get the input keys for signature 'lexical_analysis'
     data_format = lac.processor.data_format(sign_name='lexical_analysis')

demo/senta/create_module.py

@@ -48,7 +48,7 @@ def create_module():
         for_predict=True)
     hub.create_module(
         sign_arr=[sign],
-        module_dir="hub_module_senta",
+        module_dir="senta.hub_module",
         exe=exe,
         module_info="resources/module_info.yml",
         processor=processor.Processor,

demo/senta/create_module.sh

@@ -3,7 +3,7 @@ set -o nounset
 set -o errexit
 
 script_path=$(cd `dirname $0`; pwd)
-module_path=hub_module_senta
+module_path=senta.hub_module
 
 if [ -d $script_path/$module_path ]
 then

demo/senta/infer.sh

-python ../../paddlehub/commands/hub.py run hub_module_senta/ --signature sentiment_classify --config resources/test/test.yml  --input_file resources/test/test.txt
+python ../../paddlehub/commands/hub.py run senta.hub_module/ --signature sentiment_classify --config resources/test/test.yml  --input_file resources/test/test.txt

demo/senta/infer_by_code.py

@@ -4,7 +4,7 @@ import paddlehub as hub
 
 def infer_with_input_text():
     # get senta module
-    senta = hub.Module(module_dir="hub_module_senta")
+    senta = hub.Module(module_dir="senta.hub_module")
 
     test_text = ["这家餐厅很好吃", "这部电影真的很差劲"]
 
@@ -24,7 +24,7 @@ def infer_with_input_text():
 
 def infer_with_input_file():
     # get senta module
-    senta = hub.Module(module_dir="hub_module_senta")
+    senta = hub.Module(module_dir="senta.hub_module")
 
     # get the input keys for signature 'sentiment_classify'
     data_format = senta.processor.data_format(sign_name='sentiment_classify')

demo/ssd/create_module.py

@@ -47,7 +47,7 @@ def create_module():
         for_predict=True)
     hub.create_module(
         sign_arr=[sign],
-        module_dir="hub_module_ssd",
+        module_dir="ssd_mobilenet_v1_pascal.hub_module",
         module_info="resources/module_info.yml",
         exe=exe,
         processor=processor.Processor,

demo/ssd/create_module.sh

@@ -3,7 +3,7 @@ set -o nounset
 set -o errexit
 
 script_path=$(cd `dirname $0`; pwd)
-module_path=hub_module_ssd
+module_path=ssd_mobilenet_v1_pascal.hub_module
 
 if [ -d $script_path/$module_path ]
 then

demo/ssd/infer.sh

-python ../../paddlehub/commands/hub.py run hub_module_ssd/ --signature object_detection --config resources/test/test.yml  --input_file resources/test/test.txt
+python ../../paddlehub/commands/hub.py run ssd_mobilenet_v1_pascal.hub_module/ --signature object_detection --config resources/test/test.yml  --input_file resources/test/test.txt

demo/ssd/infer_by_code.py

@@ -4,7 +4,7 @@ import paddlehub as hub
 
 def infer_with_input_text():
     # get ssd module
-    ssd = hub.Module(module_dir="hub_module_ssd")
+    ssd = hub.Module(module_dir="ssd_mobilenet_v1_pascal.hub_module")
 
     test_img_path = os.path.join("resources", "test", "test_img_bird.jpg")
 
@@ -23,7 +23,7 @@ def infer_with_input_text():
 
 def infer_with_input_file():
     # get ssd module
-    ssd = hub.Module(module_dir="hub_module_ssd")
+    ssd = hub.Module(module_dir="ssd_mobilenet_v1_pascal.hub_module")
 
     # get the input keys for signature 'object_detection'
     data_format = ssd.processor.data_format(sign_name='object_detection')

paddlehub/common/utils.py

@@ -29,6 +29,27 @@ from paddlehub.module import module_desc_pb2
 from paddlehub.common.logger import logger
 
 
+def version_compare(version1, version2):
+    version1 = version1.split(".")
+    version2 = version2.split(".")
+    num = min(len(version1), len(version2))
+    for index in range(num):
+        try:
+            vn1 = int(version1[index])
+        except:
+            vn1 = 0
+        try:
+            vn2 = int(version2[index])
+        except:
+            vn2 = 0
+
+        if vn1 > vn2:
+            return True
+        elif vn1 < vn2:
+            return False
+    return len(version1) > len(version2)
+
+
 def get_platform():
     return platform.platform()
 
@@ -205,10 +226,3 @@ def get_running_device_info(config):
         dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
 
     return place, dev_count
-
-
-if __name__ == "__main__":
-    print(is_yaml_file("test.yml"))
-    print(is_csv_file("test.yml"))
-    print(is_yaml_file("test.csv"))
-    print(is_csv_file("test.csv"))

paddlehub/dataset/base_cv_dataset.py

@@ -34,7 +34,7 @@ class ImageClassificationDataset(object):
         if not os.path.exists(dataset_path):
             result, tips, dataset_path = default_downloader.download_file_and_uncompress(
                 url=url,
-                save_path=hub.dir.DATA_HOME,
+                save_path=hub.common.dir.DATA_HOME,
                 print_progress=True,
                 replace=True)
             if not result:

paddlehub/module/checker.py

@@ -21,6 +21,7 @@ import os
 import paddle
 
 from paddlehub.common.logger import logger
+from paddlehub.common.utils import version_compare
 from paddlehub.module import check_info_pb2
 from paddlehub.version import hub_version, module_proto_version
 
@@ -81,14 +82,15 @@ class ModuleChecker(object):
         return self.check_info.file_infos
 
     def check(self):
+        result = True
         self.check_info_pb_path = os.path.join(self.module_path,
                                                CHECK_INFO_PB_FILENAME)
 
         if not (os.path.exists(self.check_info_pb_path)
                 or os.path.isfile(self.check_info_pb_path)):
-            logger.error("this module lack of key documents [%s]" %
-                         CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "This module lacks core file %s" % CHECK_INFO_PB_FILENAME)
+            result = False
 
         self.check_info = check_info_pb2.CheckInfo()
         try:
@@ -97,37 +99,52 @@ class ModuleChecker(object):
                 result = self.check_info.ParseFromString(pb_string)
                 if len(pb_string) == 0 or (result is not None
                                            and result != len(pb_string)):
-                    logger.error(
-                        "the [%s] file is incomplete" % CHECK_INFO_PB_FILENAME)
-                    return False
+                    logger.warning(
+                        "File [%s] is incomplete" % CHECK_INFO_PB_FILENAME)
+                    result = False
         except Exception as e:
-            return False
+            result = False
 
         if not self.check_info.paddle_version:
-            logger.error(
-                "can't read paddle version from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning("Unable to read paddle version from [%s]" %
+                           CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.hub_version:
-            logger.error(
-                "can't read hub version from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "Unable to read hub version from [%s]" % CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.module_proto_version:
-            logger.error("can't read module pb version from [%s]" %
-                         CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning("Unable to read module pb version from [%s]" %
+                           CHECK_INFO_PB_FILENAME)
+            result = False
 
         if not self.check_info.file_infos:
-            logger.error(
-                "can't read file info from [%s]" % CHECK_INFO_PB_FILENAME)
-            return False
+            logger.warning(
+                "Unable to read file info from [%s]" % CHECK_INFO_PB_FILENAME)
+            result = False
+
+        if not self.check_module():
+            result = False
+
+        if not self.check_compatibility():
+            result = False
 
-        return self.check_module() and self.check_compatibility()
+        return result
 
     def check_compatibility(self):
-        return self._check_module_proto_version() and self._check_hub_version(
-        ) and self._check_paddle_version()
+        result = True
+        if not self._check_module_proto_version():
+            result = False
+
+        if not self._check_hub_version():
+            result = False
+
+        if not self._check_paddle_version():
+            result = False
+
+        return result
 
     def check_module(self):
         return self._check_module_integrity() and self._check_dependency()
@@ -137,34 +154,48 @@ class ModuleChecker(object):
 
     def _check_module_proto_version(self):
         if self.module_proto_version != module_proto_version:
+            logger.warning(
+                "Module description file version cannot be aligned with paddlehub version"
+            )
             return False
         return True
 
     def _check_hub_version(self):
+        if version_compare(self.hub_version, hub_version):
+            logger.warning(
+                "This Module is generated by the paddlehub with version %s, and the local paddlehub version is %s, which may cause serious incompatible bug. Please upgrade paddlehub to the latest version."
+                % (self.hub_version, hub_version))
+            return False
         return True
 
     def _check_paddle_version(self):
+        if version_compare(self.paddle_version, paddle.__version__):
+            logger.warning(
+                "This Module is generated by the paddlepaddle with version %s, and the local paddlepaddle version is %s, which may cause serious incompatible bug. Please upgrade paddlepaddle to the latest version."
+                % (self.paddle_version, paddle.__version__))
+            return False
         return True
 
     def _check_module_integrity(self):
-
+        result = True
         for file_info in self.file_infos:
             file_type = file_info.type
             file_path = file_info.file_name.replace(FILE_SEP, os.sep)
             file_path = os.path.join(self.module_path, file_path)
             if not os.path.exists(file_path):
                 if file_info.is_need:
-                    logger.error(
-                        "Module incompleted! Missing file [%s]" % file_path)
-                    return False
+                    logger.warning(
+                        "Module integrity check failed! Missing file [%s]" %
+                        file_path)
+                    result = False
             else:
                 if file_type == check_info_pb2.FILE:
                     if not os.path.isfile(file_path):
-                        logger.error("File type error %s" % file_path)
-                        return False
+                        logger.warning("File type check error %s" % file_path)
+                        result = False
 
                 if file_type == check_info_pb2.DIR:
                     if not os.path.isdir(file_path):
-                        logger.error("File type error %s" % file_path)
-                        return False
-        return True
+                        logger.warning("File type check error %s" % file_path)
+                        result = False
+        return result

paddlehub/module/module.py

@@ -191,10 +191,7 @@ class Module(object):
 
     def _init_with_module_file(self, module_dir):
         checker = ModuleChecker(module_dir)
-        if not checker.check():
-            logger.error(
-                "Module initialization failed on {}".format(module_dir))
-            exit(1)
+        checker.check()
 
         self.helper = ModuleHelper(module_dir)
         with open(self.helper.module_desc_path(), "rb") as fi: