diff --git a/paddle_hub/module/check_info.proto b/paddle_hub/module/check_info.proto new file mode 100644 index 0000000000000000000000000000000000000000..4dd119b1c96db03906f4ff45a8c20bf23abb88e8 --- /dev/null +++ b/paddle_hub/module/check_info.proto @@ -0,0 +1,39 @@ +// Copyright 2018 The Paddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +syntax = "proto3"; +option optimize_for = LITE_RUNTIME; + +package paddle_hub_check_info; + +enum FILE_TYPE { + FILE=0; + DIR=1; +} + +message FileInfo { + string file_name = 1; + FILE_TYPE type = 2; + bool is_need = 3; + string md5 = 4; + string description = 5; +} + +message CheckInfo { + string paddle_version = 1; + string hub_version = 2; + string module_proto_version = 3; + repeated FileInfo file_infos = 4; +}; diff --git a/paddle_hub/module/check_info_pb2.py b/paddle_hub/module/check_info_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..ff8acab9811c8a79303c3e906fae3c45ffc9db25 --- /dev/null +++ b/paddle_hub/module/check_info_pb2.py @@ -0,0 +1,260 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: check_info.proto + +import sys +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +DESCRIPTOR = _descriptor.FileDescriptor( + name='check_info.proto', + package='paddle_hub_check_info', + syntax='proto3', + serialized_pb=_b( + '\n\x10\x63heck_info.proto\x12\x15paddle_hub_check_info\"\x80\x01\n\x08\x46ileInfo\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12.\n\x04type\x18\x02 \x01(\x0e\x32 .paddle_hub_check_info.FILE_TYPE\x12\x0f\n\x07is_need\x18\x03 \x01(\x08\x12\x0b\n\x03md5\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\"\x8b\x01\n\tCheckInfo\x12\x16\n\x0epaddle_version\x18\x01 \x01(\t\x12\x13\n\x0bhub_version\x18\x02 \x01(\t\x12\x1c\n\x14module_proto_version\x18\x03 \x01(\t\x12\x33\n\nfile_infos\x18\x04 \x03(\x0b\x32\x1f.paddle_hub_check_info.FileInfo*\x1e\n\tFILE_TYPE\x12\x08\n\x04\x46ILE\x10\x00\x12\x07\n\x03\x44IR\x10\x01\x42\x02H\x03\x62\x06proto3' + )) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +_FILE_TYPE = _descriptor.EnumDescriptor( + name='FILE_TYPE', + full_name='paddle_hub_check_info.FILE_TYPE', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='FILE', index=0, number=0, options=None, type=None), + _descriptor.EnumValueDescriptor( + name='DIR', index=1, number=1, options=None, type=None), + ], + containing_type=None, + options=None, + serialized_start=316, + serialized_end=346, +) +_sym_db.RegisterEnumDescriptor(_FILE_TYPE) + +FILE_TYPE = enum_type_wrapper.EnumTypeWrapper(_FILE_TYPE) +FILE = 0 +DIR = 1 + +_FILEINFO = _descriptor.Descriptor( + name='FileInfo', + full_name='paddle_hub_check_info.FileInfo', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='file_name', + full_name='paddle_hub_check_info.FileInfo.file_name', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', + full_name='paddle_hub_check_info.FileInfo.type', + index=1, + number=2, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='is_need', + full_name='paddle_hub_check_info.FileInfo.is_need', + index=2, + number=3, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='md5', + full_name='paddle_hub_check_info.FileInfo.md5', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='description', + full_name='paddle_hub_check_info.FileInfo.description', + index=4, + number=5, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=44, + serialized_end=172, +) + +_CHECKINFO = _descriptor.Descriptor( + name='CheckInfo', + full_name='paddle_hub_check_info.CheckInfo', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='paddle_version', + full_name='paddle_hub_check_info.CheckInfo.paddle_version', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='hub_version', + full_name='paddle_hub_check_info.CheckInfo.hub_version', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='module_proto_version', + full_name='paddle_hub_check_info.CheckInfo.module_proto_version', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='file_infos', + full_name='paddle_hub_check_info.CheckInfo.file_infos', + index=3, + number=4, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=175, + serialized_end=314, +) + +_FILEINFO.fields_by_name['type'].enum_type = _FILE_TYPE +_CHECKINFO.fields_by_name['file_infos'].message_type = _FILEINFO +DESCRIPTOR.message_types_by_name['FileInfo'] = _FILEINFO +DESCRIPTOR.message_types_by_name['CheckInfo'] = _CHECKINFO +DESCRIPTOR.enum_types_by_name['FILE_TYPE'] = _FILE_TYPE + +FileInfo = _reflection.GeneratedProtocolMessageType( + 'FileInfo', + (_message.Message, ), + dict( + DESCRIPTOR=_FILEINFO, + __module__='check_info_pb2' + # @@protoc_insertion_point(class_scope:paddle_hub_check_info.FileInfo) + )) +_sym_db.RegisterMessage(FileInfo) + +CheckInfo = _reflection.GeneratedProtocolMessageType( + 'CheckInfo', + (_message.Message, ), + dict( + DESCRIPTOR=_CHECKINFO, + __module__='check_info_pb2' + # @@protoc_insertion_point(class_scope:paddle_hub_check_info.CheckInfo) + )) +_sym_db.RegisterMessage(CheckInfo) + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), + _b('H\003')) +# @@protoc_insertion_point(module_scope) diff --git a/paddle_hub/module/checker.py b/paddle_hub/module/checker.py new file mode 100644 index 0000000000000000000000000000000000000000..2ca93b19f032ed946609215239fe1844e6f6cf8b --- /dev/null +++ b/paddle_hub/module/checker.py @@ -0,0 +1,178 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from paddle_hub.tools.logger import logger +from paddle_hub.module import check_info_pb2 +from paddle_hub.version import hub_version, module_proto_version +import os +import paddle + +# check info +CHECK_INFO_PB_FILENAME = "check_info.pb" +FILE_SEP = "/" + + +class ModuleChecker: + def __init__(self, module_path): + self.module_path = module_path + + def generate_check_info(self): + check_info = check_info_pb2.CheckInfo() + check_info.paddle_version = paddle.__version__ + check_info.hub_version = hub_version + check_info.module_proto_version = module_proto_version + file_infos = check_info.file_infos + file_list = [file for file in os.listdir(self.module_path)] + while file_list: + file = file_list[0] + file_list = file_list[1:] + abs_path = os.path.join(self.module_path, file) + if os.path.isdir(abs_path): + for sub_file in os.listdir(abs_path): + sub_file = os.path.join(file, sub_file) + file_list.append(sub_file) + file_info = file_infos.add() + file_info.file_name = file + file.replace(os.sep, FILE_SEP) + file_info.type = check_info_pb2.DIR + file_info.is_need = True + else: + file.replace(os.sep, FILE_SEP) + file_info = file_infos.add() + file_info.file_name = file + file_info.type = check_info_pb2.FILE + file_info.is_need = True + + with open(os.path.join(self.module_path, CHECK_INFO_PB_FILENAME), + "wb") as fi: + fi.write(check_info.SerializeToString()) + + @property + def module_proto_version(self): + return self.check_info.module_proto_version + + @property + def hub_version(self): + return self.check_info.hub_version + + @property + def paddle_version(self): + return self.check_info.paddle_version + + @property + def file_infos(self): + return self.check_info.file_infos + + def check(self): + self.check_info_pb_path = os.path.join(self.module_path, + CHECK_INFO_PB_FILENAME) + + if not (os.path.exists(self.check_info_pb_path) + or os.path.isfile(self.check_info_pb_path)): + logger.error("this module lack of key documents [%s]" % + CHECK_INFO_PB_FILENAME) + return False + + self.check_info = check_info_pb2.CheckInfo() + try: + with open(self.check_info_pb_path, "rb") as fi: + pb_string = fi.read() + result = self.check_info.ParseFromString(pb_string) + if len(pb_string) == 0 or (result is not None + and result != len(pb_string)): + logger.error( + "the [%s] file is incomplete" % CHECK_INFO_PB_FILENAME) + return False + except Exception as e: + return False + + if not self.check_info.paddle_version: + logger.error( + "can't read paddle version from [%s]" % CHECK_INFO_PB_FILENAME) + return False + + if not self.check_info.hub_version: + logger.error( + "can't read hub version from [%s]" % CHECK_INFO_PB_FILENAME) + return False + + if not self.check_info.module_proto_version: + logger.error("can't read module pb version from [%s]" % + CHECK_INFO_PB_FILENAME) + return False + + if not self.check_info.file_infos: + logger.error( + "can't read file info from [%s]" % CHECK_INFO_PB_FILENAME) + return False + + return self.check_module() and self.check_compatibility() + + def check_compatibility(self): + return self._check_module_proto_version() and self._check_hub_version( + ) and self._check_paddle_version() + + def check_module(self): + return self._check_module_integrity() and self._check_dependency() + + def _check_dependency(self): + return True + + def _check_module_proto_version(self): + if self.module_proto_version != module_proto_version: + return False + return True + + def _check_hub_version(self): + return True + + def _check_paddle_version(self): + return True + + def _check_module_integrity(self): + + for file_info in self.file_infos: + file_type = file_info.type + file_path = file_info.file_name.replace(FILE_SEP, os.sep) + file_path = os.path.join(self.module_path, file_path) + if not os.path.exists(file_path): + if file_info.is_need: + logger.error( + "module lack of necessary file [%s]" % file_path) + return False + else: + if file_type == check_info_pb2.FILE: + if not os.path.isfile(file_path): + logger.error("file type error %s" % file_path) + return False + + if file_type == check_info_pb2.DIR: + if not os.path.isdir(file_path): + logger.error("file type error %s" % file_path) + return False + return True + + +if __name__ == "__main__": + # check_info = check_info_pb2.CheckInfo() + # check_info.paddle_version = "1" + # check_info.hub_version = "1" + # check_info.module_proto_version = "1" + # with open(os.path.join(".", CHECK_INFO_PB_FILENAME), "wb") as fi: + # fi.write(check_info.SerializeToString()) + check_info = ModuleChecker( + "/home/wuzewu/code/PaddleHub/demo/object-detection/hub_module_ssd") + print(check_info.check()) diff --git a/paddle_hub/module/module.py b/paddle_hub/module/module.py index f1d0a310fd69ff7ae5d958555b36659356f36e19..5551bbc5ae79ed8510fcde5a25c947b7ae3bc239 100644 --- a/paddle_hub/module/module.py +++ b/paddle_hub/module/module.py @@ -20,7 +20,9 @@ from paddle_hub.tools.logger import logger from paddle_hub.tools.downloader import default_downloader from paddle_hub.tools import paddle_helper from paddle_hub.module import module_desc_pb2 +from paddle_hub.module import check_info_pb2 from paddle_hub.module.signature import Signature, create_signature +from paddle_hub.module.checker import ModuleChecker from paddle_hub.data.reader import yaml_reader from paddle_hub import version from paddle_hub.module.base_processor import BaseProcessor @@ -156,6 +158,11 @@ class Module: self.assets.append(filepath) def _init_with_module_file(self, module_dir): + checker = ModuleChecker(module_dir) + if not checker.check(): + logger.error("module check fail") + exit(1) + self.helper = ModuleHelper(module_dir) with open(self.helper.module_desc_path(), "rb") as fi: self.desc.ParseFromString(fi.read()) @@ -464,7 +471,7 @@ class Module: # create module pb module_desc = module_desc_pb2.ModuleDesc() logger.info("hub version is %s" % version.hub_version) - logger.info("proto version is %s" % version.proto_version) + logger.info("module proto version is %s" % version.module_proto_version) logger.info("paddle version is %s" % paddle.__version__) feeded_var_names = [ @@ -530,3 +537,7 @@ class Module: # create assets self._dump_assets() + + # create check info + checker = ModuleChecker(self.helper.module_dir) + checker.generate_check_info() diff --git a/paddle_hub/version.py b/paddle_hub/version.py index 2423ddeba29b8a6eeef2c183987978a7e4c71b6c..ae06a11cef476f099ad90cf2144d410f3e40accd 100644 --- a/paddle_hub/version.py +++ b/paddle_hub/version.py @@ -13,4 +13,4 @@ # limitations under the License. """ Store PaddleHub version string """ hub_version = "0.1.0.dev" -proto_version = "0.1.0" +module_proto_version = "0.1.0" diff --git a/scripts/gen_proto.sh b/scripts/gen_proto.sh index 781698996e4c51bb6d892b0dee029a49c69361c1..6c4481b455faa19481a93551abca64f50836f842 100755 --- a/scripts/gen_proto.sh +++ b/scripts/gen_proto.sh @@ -1,2 +1,3 @@ #/bin/bash protoc -I=../paddle_hub/module --python_out=../paddle_hub/module ../paddle_hub/module/module_desc.proto +protoc -I=../paddle_hub/module --python_out=../paddle_hub/module ../paddle_hub/module/check_info.proto