diff --git a/demo/image-classification/create_module.py b/demo/image-classification/create_module.py index 3015066b9072f28268be2831a487e70a66f1e2e6..3f4dffbf03f1e42b33ec23ccf0197fa1ebf0e796 100644 --- a/demo/image-classification/create_module.py +++ b/demo/image-classification/create_module.py @@ -56,7 +56,7 @@ def create_module(args): "feature_map", inputs=[image], outputs=[feature_map]) hub.create_module( sign_arr=[sign1, sign2], - module_dir="hub_module_" + args.model, + module_dir=args.model + ".hub_module", module_info="resources/module_info.yml", processor=processor.Processor, assets=assets) diff --git a/demo/image-classification/create_module.sh b/demo/image-classification/create_module.sh index 04443000a2cd4cf5e316c0b4b964e79b417b09b5..cd288406169932c4c147c37203dc6ed20d63f843 100644 --- a/demo/image-classification/create_module.sh +++ b/demo/image-classification/create_module.sh @@ -16,7 +16,7 @@ do done script_path=$(cd `dirname $0`; pwd) -module_path=hub_module_${model_name} +module_path=${model_name}.hub_module if [ -d $script_path/$module_path ] then diff --git a/demo/image-classification/infer.sh b/demo/image-classification/infer.sh index 6aff1d777eeb7cb82696e16ab5acea73ba34a289..2f36d86a46736e51f75e00676aa1a08cd20833d1 100644 --- a/demo/image-classification/infer.sh +++ b/demo/image-classification/infer.sh @@ -1 +1 @@ -python ../../paddlehub/commands/hub.py run hub_module_ResNet50/ --signature classification --config resources/test/test.yml --dataset resources/test/test.csv +python ../../paddlehub/commands/hub.py run ResNet50.hub_module/ --signature classification --config resources/test/test.yml --dataset resources/test/test.csv diff --git a/demo/image-classification/resources/module_info.yml b/demo/image-classification/resources/module_info.yml index 255e7fecb2b987cb78916cd9afc64944fa6cb9a9..7eae4b4f4acb953c98fb2505be7ae46cd57700e6 100644 --- a/demo/image-classification/resources/module_info.yml +++ b/demo/image-classification/resources/module_info.yml @@ -1,4 +1,4 @@ -name: Resnet50 +name: resnet_v2_50_imagenet type: CV/classification author: paddlepaddle author_email: paddle-dev@baidu.com diff --git a/demo/image-classification/retrain.py b/demo/image-classification/retrain.py index 77350776878ad0a7fcc82b8eb3809839344c5ee0..1e6e47ce41f83e649955c5ee134f44066f2df431 100644 --- a/demo/image-classification/retrain.py +++ b/demo/image-classification/retrain.py @@ -5,11 +5,11 @@ import paddlehub as hub def train(): - resnet_module = hub.Module(module_dir="hub_module_ResNet50") + resnet_module = hub.Module(module_dir="ResNet50.hub_module") input_dict, output_dict, program = resnet_module.context( sign_name="feature_map", trainable=True) dataset = hub.dataset.Flowers() - data_reader = hub.ImageClassificationReader( + data_reader = hub.reader.ImageClassificationReader( image_width=224, image_height=224, dataset=dataset) with fluid.program_guard(program): label = fluid.layers.data(name="label", dtype="int64", shape=[1]) @@ -24,7 +24,7 @@ def train(): feed_list = [img.name, label.name] - task = hub.append_mlp_classifier( + task = hub.create_img_classification_task( feature=feature_map, label=label, num_classes=dataset.num_labels) hub.finetune_and_eval( task, feed_list=feed_list, data_reader=data_reader, config=config) diff --git a/demo/lac/create_module.py b/demo/lac/create_module.py index a7710e37df261171931da8d18447bb4f26a482f3..18a2a4dda2e614eb21de47c4b76969436eb9228c 100644 --- a/demo/lac/create_module.py +++ b/demo/lac/create_module.py @@ -35,7 +35,7 @@ def create_module(): # assets assets = [word_dict_path, label_dict_path, word_rep_dict_path] - # create a module and save as hub_module_lac + # create a module and save as lac.hub_module sign = hub.create_signature( name="lexical_analysis", inputs=[word], @@ -43,7 +43,7 @@ def create_module(): for_predict=True) hub.create_module( sign_arr=[sign], - module_dir="hub_module_lac", + module_dir="lac.hub_module", exe=exe, module_info="resources/module_info.yml", processor=processor.Processor, diff --git a/demo/lac/create_module.sh b/demo/lac/create_module.sh index ecad3685de15f31f98d2f28bfad17ee1433c8698..760e0af87ef3b013cc01febb2cb17c77a7afc590 100644 --- a/demo/lac/create_module.sh +++ b/demo/lac/create_module.sh @@ -3,7 +3,7 @@ set -o nounset set -o errexit script_path=$(cd `dirname $0`; pwd) -module_path=hub_module_lac +module_path=lac.hub_module if [ -d $script_path/$module_path ] then diff --git a/demo/lac/infer.sh b/demo/lac/infer.sh index c771bc3c73a28dab0b0c8615cb661437d67b9917..bc45d879a6b2b1e88b6692ed02891b11c056476f 100644 --- a/demo/lac/infer.sh +++ b/demo/lac/infer.sh @@ -1 +1 @@ -python ../../paddlehub/commands/hub.py run hub_module_lac/ --signature lexical_analysis --config resources/test/test.yml --input_file resources/test/test.txt +python ../../paddlehub/commands/hub.py run lac.hub_module/ --signature lexical_analysis --config resources/test/test.yml --input_file resources/test/test.txt diff --git a/demo/lac/infer_by_code.py b/demo/lac/infer_by_code.py index acbb9881fab5a7077831a799584228f78d0f1d54..34680185665b9c2ce9db46c662f02e4d38de6c1b 100644 --- a/demo/lac/infer_by_code.py +++ b/demo/lac/infer_by_code.py @@ -4,7 +4,7 @@ import paddlehub as hub def infer_with_input_text(): # get lac module - lac = hub.Module(module_dir="hub_module_lac") + lac = hub.Module(module_dir="lac.hub_module") test_text = ["今天是个好日子", "天气预报说今天要下雨", "下一班地铁马上就要到了"] @@ -24,7 +24,7 @@ def infer_with_input_text(): def infer_with_input_file(): # get lac module - lac = hub.Module(module_dir="hub_module_lac") + lac = hub.Module(module_dir="lac.hub_module") # get the input keys for signature 'lexical_analysis' data_format = lac.processor.data_format(sign_name='lexical_analysis') diff --git a/demo/senta/create_module.py b/demo/senta/create_module.py index cc500410ed119ddf60449d0ca0cb26b8f8e6ae46..f829b58bfa0077d02b289e6e223af8694e79755c 100644 --- a/demo/senta/create_module.py +++ b/demo/senta/create_module.py @@ -48,7 +48,7 @@ def create_module(): for_predict=True) hub.create_module( sign_arr=[sign], - module_dir="hub_module_senta", + module_dir="senta.hub_module", exe=exe, module_info="resources/module_info.yml", processor=processor.Processor, diff --git a/demo/senta/create_module.sh b/demo/senta/create_module.sh index 09e51e21a6940dbcf4c98f4a146b9bda23f27c86..cfbad8d33598e335082b16aa8cc383dfbcebf2cc 100644 --- a/demo/senta/create_module.sh +++ b/demo/senta/create_module.sh @@ -3,7 +3,7 @@ set -o nounset set -o errexit script_path=$(cd `dirname $0`; pwd) -module_path=hub_module_senta +module_path=senta.hub_module if [ -d $script_path/$module_path ] then diff --git a/demo/senta/infer.sh b/demo/senta/infer.sh index 661cad1c8e9e80ad9b9a29880456948fb264e0b5..80ec1b278365904536614276532c7c645724bc07 100644 --- a/demo/senta/infer.sh +++ b/demo/senta/infer.sh @@ -1 +1 @@ -python ../../paddlehub/commands/hub.py run hub_module_senta/ --signature sentiment_classify --config resources/test/test.yml --input_file resources/test/test.txt +python ../../paddlehub/commands/hub.py run senta.hub_module/ --signature sentiment_classify --config resources/test/test.yml --input_file resources/test/test.txt diff --git a/demo/senta/infer_by_code.py b/demo/senta/infer_by_code.py index 660feebc0b6647539704701af7cce08580a05749..bba15c2135d21da3b7c3f80b723819d7917f819b 100644 --- a/demo/senta/infer_by_code.py +++ b/demo/senta/infer_by_code.py @@ -4,7 +4,7 @@ import paddlehub as hub def infer_with_input_text(): # get senta module - senta = hub.Module(module_dir="hub_module_senta") + senta = hub.Module(module_dir="senta.hub_module") test_text = ["这家餐厅很好吃", "这部电影真的很差劲"] @@ -24,7 +24,7 @@ def infer_with_input_text(): def infer_with_input_file(): # get senta module - senta = hub.Module(module_dir="hub_module_senta") + senta = hub.Module(module_dir="senta.hub_module") # get the input keys for signature 'sentiment_classify' data_format = senta.processor.data_format(sign_name='sentiment_classify') diff --git a/demo/ssd/create_module.py b/demo/ssd/create_module.py index c35c6b1d6192b34e11acf45d6c84e6cb943d36a1..750997dda0a72ef7ca8ce47a9f5f985a7e66482f 100644 --- a/demo/ssd/create_module.py +++ b/demo/ssd/create_module.py @@ -47,7 +47,7 @@ def create_module(): for_predict=True) hub.create_module( sign_arr=[sign], - module_dir="hub_module_ssd", + module_dir="ssd_mobilenet_v1_pascal.hub_module", module_info="resources/module_info.yml", exe=exe, processor=processor.Processor, diff --git a/demo/ssd/create_module.sh b/demo/ssd/create_module.sh index 13c9d573c3595285fa3aedf1cd4ddf9dd8a1de79..1e61612d6886f08b18297eaccfd48eb7ad365123 100644 --- a/demo/ssd/create_module.sh +++ b/demo/ssd/create_module.sh @@ -3,7 +3,7 @@ set -o nounset set -o errexit script_path=$(cd `dirname $0`; pwd) -module_path=hub_module_ssd +module_path=ssd_mobilenet_v1_pascal.hub_module if [ -d $script_path/$module_path ] then diff --git a/demo/ssd/infer.sh b/demo/ssd/infer.sh index b461ffe63ded8a9a4756ff5132ab0d2c081104b3..dce4a63162edff6544b0c164b7538c15f4096d6c 100644 --- a/demo/ssd/infer.sh +++ b/demo/ssd/infer.sh @@ -1 +1 @@ -python ../../paddlehub/commands/hub.py run hub_module_ssd/ --signature object_detection --config resources/test/test.yml --input_file resources/test/test.txt +python ../../paddlehub/commands/hub.py run ssd_mobilenet_v1_pascal.hub_module/ --signature object_detection --config resources/test/test.yml --input_file resources/test/test.txt diff --git a/demo/ssd/infer_by_code.py b/demo/ssd/infer_by_code.py index fff80a0c1527955f19726952f8fdd0702b02b43d..977b8c5c79ad6a785e765d6a80df66836efc2b0f 100644 --- a/demo/ssd/infer_by_code.py +++ b/demo/ssd/infer_by_code.py @@ -4,7 +4,7 @@ import paddlehub as hub def infer_with_input_text(): # get ssd module - ssd = hub.Module(module_dir="hub_module_ssd") + ssd = hub.Module(module_dir="ssd_mobilenet_v1_pascal.hub_module") test_img_path = os.path.join("resources", "test", "test_img_bird.jpg") @@ -23,7 +23,7 @@ def infer_with_input_text(): def infer_with_input_file(): # get ssd module - ssd = hub.Module(module_dir="hub_module_ssd") + ssd = hub.Module(module_dir="ssd_mobilenet_v1_pascal.hub_module") # get the input keys for signature 'object_detection' data_format = ssd.processor.data_format(sign_name='object_detection') diff --git a/paddlehub/common/utils.py b/paddlehub/common/utils.py index 92fd470bc589618d8ec71cdf83961469c4809bab..641c6a59fa1360514877be2b357f4d9f004bca63 100644 --- a/paddlehub/common/utils.py +++ b/paddlehub/common/utils.py @@ -29,6 +29,27 @@ from paddlehub.module import module_desc_pb2 from paddlehub.common.logger import logger +def version_compare(version1, version2): + version1 = version1.split(".") + version2 = version2.split(".") + num = min(len(version1), len(version2)) + for index in range(num): + try: + vn1 = int(version1[index]) + except: + vn1 = 0 + try: + vn2 = int(version2[index]) + except: + vn2 = 0 + + if vn1 > vn2: + return True + elif vn1 < vn2: + return False + return len(version1) > len(version2) + + def get_platform(): return platform.platform() @@ -205,10 +226,3 @@ def get_running_device_info(config): dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) return place, dev_count - - -if __name__ == "__main__": - print(is_yaml_file("test.yml")) - print(is_csv_file("test.yml")) - print(is_yaml_file("test.csv")) - print(is_csv_file("test.csv")) diff --git a/paddlehub/dataset/base_cv_dataset.py b/paddlehub/dataset/base_cv_dataset.py index 053dda3b85467da0db34e5f3440872bdd36c8ca6..7e0b687d1d93ff91ee15f51a8218e6266a8147e2 100644 --- a/paddlehub/dataset/base_cv_dataset.py +++ b/paddlehub/dataset/base_cv_dataset.py @@ -34,7 +34,7 @@ class ImageClassificationDataset(object): if not os.path.exists(dataset_path): result, tips, dataset_path = default_downloader.download_file_and_uncompress( url=url, - save_path=hub.dir.DATA_HOME, + save_path=hub.common.dir.DATA_HOME, print_progress=True, replace=True) if not result: diff --git a/paddlehub/module/checker.py b/paddlehub/module/checker.py index bab8ff6fb30fc386f8527c618ddbf3c68cbb715f..2e10a0dddab1605250f035ba8e09816885296b9c 100644 --- a/paddlehub/module/checker.py +++ b/paddlehub/module/checker.py @@ -21,6 +21,7 @@ import os import paddle from paddlehub.common.logger import logger +from paddlehub.common.utils import version_compare from paddlehub.module import check_info_pb2 from paddlehub.version import hub_version, module_proto_version @@ -81,14 +82,15 @@ class ModuleChecker(object): return self.check_info.file_infos def check(self): + result = True self.check_info_pb_path = os.path.join(self.module_path, CHECK_INFO_PB_FILENAME) if not (os.path.exists(self.check_info_pb_path) or os.path.isfile(self.check_info_pb_path)): - logger.error("this module lack of key documents [%s]" % - CHECK_INFO_PB_FILENAME) - return False + logger.warning( + "This module lacks core file %s" % CHECK_INFO_PB_FILENAME) + result = False self.check_info = check_info_pb2.CheckInfo() try: @@ -97,37 +99,52 @@ class ModuleChecker(object): result = self.check_info.ParseFromString(pb_string) if len(pb_string) == 0 or (result is not None and result != len(pb_string)): - logger.error( - "the [%s] file is incomplete" % CHECK_INFO_PB_FILENAME) - return False + logger.warning( + "File [%s] is incomplete" % CHECK_INFO_PB_FILENAME) + result = False except Exception as e: - return False + result = False if not self.check_info.paddle_version: - logger.error( - "can't read paddle version from [%s]" % CHECK_INFO_PB_FILENAME) - return False + logger.warning("Unable to read paddle version from [%s]" % + CHECK_INFO_PB_FILENAME) + result = False if not self.check_info.hub_version: - logger.error( - "can't read hub version from [%s]" % CHECK_INFO_PB_FILENAME) - return False + logger.warning( + "Unable to read hub version from [%s]" % CHECK_INFO_PB_FILENAME) + result = False if not self.check_info.module_proto_version: - logger.error("can't read module pb version from [%s]" % - CHECK_INFO_PB_FILENAME) - return False + logger.warning("Unable to read module pb version from [%s]" % + CHECK_INFO_PB_FILENAME) + result = False if not self.check_info.file_infos: - logger.error( - "can't read file info from [%s]" % CHECK_INFO_PB_FILENAME) - return False + logger.warning( + "Unable to read file info from [%s]" % CHECK_INFO_PB_FILENAME) + result = False + + if not self.check_module(): + result = False + + if not self.check_compatibility(): + result = False - return self.check_module() and self.check_compatibility() + return result def check_compatibility(self): - return self._check_module_proto_version() and self._check_hub_version( - ) and self._check_paddle_version() + result = True + if not self._check_module_proto_version(): + result = False + + if not self._check_hub_version(): + result = False + + if not self._check_paddle_version(): + result = False + + return result def check_module(self): return self._check_module_integrity() and self._check_dependency() @@ -137,34 +154,48 @@ class ModuleChecker(object): def _check_module_proto_version(self): if self.module_proto_version != module_proto_version: + logger.warning( + "Module description file version cannot be aligned with paddlehub version" + ) return False return True def _check_hub_version(self): + if version_compare(self.hub_version, hub_version): + logger.warning( + "This Module is generated by the paddlehub with version %s, and the local paddlehub version is %s, which may cause serious incompatible bug. Please upgrade paddlehub to the latest version." + % (self.hub_version, hub_version)) + return False return True def _check_paddle_version(self): + if version_compare(self.paddle_version, paddle.__version__): + logger.warning( + "This Module is generated by the paddlepaddle with version %s, and the local paddlepaddle version is %s, which may cause serious incompatible bug. Please upgrade paddlepaddle to the latest version." + % (self.paddle_version, paddle.__version__)) + return False return True def _check_module_integrity(self): - + result = True for file_info in self.file_infos: file_type = file_info.type file_path = file_info.file_name.replace(FILE_SEP, os.sep) file_path = os.path.join(self.module_path, file_path) if not os.path.exists(file_path): if file_info.is_need: - logger.error( - "Module incompleted! Missing file [%s]" % file_path) - return False + logger.warning( + "Module integrity check failed! Missing file [%s]" % + file_path) + result = False else: if file_type == check_info_pb2.FILE: if not os.path.isfile(file_path): - logger.error("File type error %s" % file_path) - return False + logger.warning("File type check error %s" % file_path) + result = False if file_type == check_info_pb2.DIR: if not os.path.isdir(file_path): - logger.error("File type error %s" % file_path) - return False - return True + logger.warning("File type check error %s" % file_path) + result = False + return result diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index 101e1262810f2b2dbc93e72f4f628fb1cd74be89..c5512d5165f1d2d9ad96355a4a995e610cc0aa2e 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -191,10 +191,7 @@ class Module(object): def _init_with_module_file(self, module_dir): checker = ModuleChecker(module_dir) - if not checker.check(): - logger.error( - "Module initialization failed on {}".format(module_dir)) - exit(1) + checker.check() self.helper = ModuleHelper(module_dir) with open(self.helper.module_desc_path(), "rb") as fi: