add module checker

paddle_hub/module/check_info.proto

+// Copyright 2018 The Paddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+syntax = "proto3";
+option optimize_for = LITE_RUNTIME;
+
+package paddle_hub_check_info;
+
+enum FILE_TYPE {
+  FILE=0;
+  DIR=1;
+}
+
+message FileInfo {
+  string file_name = 1;
+  FILE_TYPE type = 2;
+  bool is_need = 3;
+  string md5 = 4;
+  string description = 5;
+}
+
+message CheckInfo {
+  string paddle_version = 1;
+  string hub_version = 2;
+  string module_proto_version = 3;
+  repeated FileInfo file_infos = 4;
+};

paddle_hub/module/check_info_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: check_info.proto
+
+import sys
+_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+    name='check_info.proto',
+    package='paddle_hub_check_info',
+    syntax='proto3',
+    serialized_pb=_b(
+        '\n\x10\x63heck_info.proto\x12\x15paddle_hub_check_info\"\x80\x01\n\x08\x46ileInfo\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12.\n\x04type\x18\x02 \x01(\x0e\x32 .paddle_hub_check_info.FILE_TYPE\x12\x0f\n\x07is_need\x18\x03 \x01(\x08\x12\x0b\n\x03md5\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\"\x8b\x01\n\tCheckInfo\x12\x16\n\x0epaddle_version\x18\x01 \x01(\t\x12\x13\n\x0bhub_version\x18\x02 \x01(\t\x12\x1c\n\x14module_proto_version\x18\x03 \x01(\t\x12\x33\n\nfile_infos\x18\x04 \x03(\x0b\x32\x1f.paddle_hub_check_info.FileInfo*\x1e\n\tFILE_TYPE\x12\x08\n\x04\x46ILE\x10\x00\x12\x07\n\x03\x44IR\x10\x01\x42\x02H\x03\x62\x06proto3'
+    ))
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+_FILE_TYPE = _descriptor.EnumDescriptor(
+    name='FILE_TYPE',
+    full_name='paddle_hub_check_info.FILE_TYPE',
+    filename=None,
+    file=DESCRIPTOR,
+    values=[
+        _descriptor.EnumValueDescriptor(
+            name='FILE', index=0, number=0, options=None, type=None),
+        _descriptor.EnumValueDescriptor(
+            name='DIR', index=1, number=1, options=None, type=None),
+    ],
+    containing_type=None,
+    options=None,
+    serialized_start=316,
+    serialized_end=346,
+)
+_sym_db.RegisterEnumDescriptor(_FILE_TYPE)
+
+FILE_TYPE = enum_type_wrapper.EnumTypeWrapper(_FILE_TYPE)
+FILE = 0
+DIR = 1
+
+_FILEINFO = _descriptor.Descriptor(
+    name='FileInfo',
+    full_name='paddle_hub_check_info.FileInfo',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='file_name',
+            full_name='paddle_hub_check_info.FileInfo.file_name',
+            index=0,
+            number=1,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='type',
+            full_name='paddle_hub_check_info.FileInfo.type',
+            index=1,
+            number=2,
+            type=14,
+            cpp_type=8,
+            label=1,
+            has_default_value=False,
+            default_value=0,
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='is_need',
+            full_name='paddle_hub_check_info.FileInfo.is_need',
+            index=2,
+            number=3,
+            type=8,
+            cpp_type=7,
+            label=1,
+            has_default_value=False,
+            default_value=False,
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='md5',
+            full_name='paddle_hub_check_info.FileInfo.md5',
+            index=3,
+            number=4,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='description',
+            full_name='paddle_hub_check_info.FileInfo.description',
+            index=4,
+            number=5,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+    ],
+    extensions=[],
+    nested_types=[],
+    enum_types=[],
+    options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[],
+    serialized_start=44,
+    serialized_end=172,
+)
+
+_CHECKINFO = _descriptor.Descriptor(
+    name='CheckInfo',
+    full_name='paddle_hub_check_info.CheckInfo',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='paddle_version',
+            full_name='paddle_hub_check_info.CheckInfo.paddle_version',
+            index=0,
+            number=1,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='hub_version',
+            full_name='paddle_hub_check_info.CheckInfo.hub_version',
+            index=1,
+            number=2,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='module_proto_version',
+            full_name='paddle_hub_check_info.CheckInfo.module_proto_version',
+            index=2,
+            number=3,
+            type=9,
+            cpp_type=9,
+            label=1,
+            has_default_value=False,
+            default_value=_b("").decode('utf-8'),
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+        _descriptor.FieldDescriptor(
+            name='file_infos',
+            full_name='paddle_hub_check_info.CheckInfo.file_infos',
+            index=3,
+            number=4,
+            type=11,
+            cpp_type=10,
+            label=3,
+            has_default_value=False,
+            default_value=[],
+            message_type=None,
+            enum_type=None,
+            containing_type=None,
+            is_extension=False,
+            extension_scope=None,
+            options=None),
+    ],
+    extensions=[],
+    nested_types=[],
+    enum_types=[],
+    options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[],
+    serialized_start=175,
+    serialized_end=314,
+)
+
+_FILEINFO.fields_by_name['type'].enum_type = _FILE_TYPE
+_CHECKINFO.fields_by_name['file_infos'].message_type = _FILEINFO
+DESCRIPTOR.message_types_by_name['FileInfo'] = _FILEINFO
+DESCRIPTOR.message_types_by_name['CheckInfo'] = _CHECKINFO
+DESCRIPTOR.enum_types_by_name['FILE_TYPE'] = _FILE_TYPE
+
+FileInfo = _reflection.GeneratedProtocolMessageType(
+    'FileInfo',
+    (_message.Message, ),
+    dict(
+        DESCRIPTOR=_FILEINFO,
+        __module__='check_info_pb2'
+        # @@protoc_insertion_point(class_scope:paddle_hub_check_info.FileInfo)
+    ))
+_sym_db.RegisterMessage(FileInfo)
+
+CheckInfo = _reflection.GeneratedProtocolMessageType(
+    'CheckInfo',
+    (_message.Message, ),
+    dict(
+        DESCRIPTOR=_CHECKINFO,
+        __module__='check_info_pb2'
+        # @@protoc_insertion_point(class_scope:paddle_hub_check_info.CheckInfo)
+    ))
+_sym_db.RegisterMessage(CheckInfo)
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(),
+                                                _b('H\003'))
+# @@protoc_insertion_point(module_scope)

paddle_hub/module/checker.py

+# Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from paddle_hub.tools.logger import logger
+from paddle_hub.module import check_info_pb2
+from paddle_hub.version import hub_version, module_proto_version
+import os
+import paddle
+
+# check info
+CHECK_INFO_PB_FILENAME = "check_info.pb"
+FILE_SEP = "/"
+
+
+class ModuleChecker:
+    def __init__(self, module_path):
+        self.module_path = module_path
+
+    def generate_check_info(self):
+        check_info = check_info_pb2.CheckInfo()
+        check_info.paddle_version = paddle.__version__
+        check_info.hub_version = hub_version
+        check_info.module_proto_version = module_proto_version
+        file_infos = check_info.file_infos
+        file_list = [file for file in os.listdir(self.module_path)]
+        while file_list:
+            file = file_list[0]
+            file_list = file_list[1:]
+            abs_path = os.path.join(self.module_path, file)
+            if os.path.isdir(abs_path):
+                for sub_file in os.listdir(abs_path):
+                    sub_file = os.path.join(file, sub_file)
+                    file_list.append(sub_file)
+                file_info = file_infos.add()
+                file_info.file_name = file
+                file.replace(os.sep, FILE_SEP)
+                file_info.type = check_info_pb2.DIR
+                file_info.is_need = True
+            else:
+                file.replace(os.sep, FILE_SEP)
+                file_info = file_infos.add()
+                file_info.file_name = file
+                file_info.type = check_info_pb2.FILE
+                file_info.is_need = True
+
+        with open(os.path.join(self.module_path, CHECK_INFO_PB_FILENAME),
+                  "wb") as fi:
+            fi.write(check_info.SerializeToString())
+
+    @property
+    def module_proto_version(self):
+        return self.check_info.module_proto_version
+
+    @property
+    def hub_version(self):
+        return self.check_info.hub_version
+
+    @property
+    def paddle_version(self):
+        return self.check_info.paddle_version
+
+    @property
+    def file_infos(self):
+        return self.check_info.file_infos
+
+    def check(self):
+        self.check_info_pb_path = os.path.join(self.module_path,
+                                               CHECK_INFO_PB_FILENAME)
+
+        if not (os.path.exists(self.check_info_pb_path)
+                or os.path.isfile(self.check_info_pb_path)):
+            logger.error("this module lack of key documents [%s]" %
+                         CHECK_INFO_PB_FILENAME)
+            return False
+
+        self.check_info = check_info_pb2.CheckInfo()
+        try:
+            with open(self.check_info_pb_path, "rb") as fi:
+                pb_string = fi.read()
+                result = self.check_info.ParseFromString(pb_string)
+                if len(pb_string) == 0 or (result is not None
+                                           and result != len(pb_string)):
+                    logger.error(
+                        "the [%s] file is incomplete" % CHECK_INFO_PB_FILENAME)
+                    return False
+        except Exception as e:
+            return False
+
+        if not self.check_info.paddle_version:
+            logger.error(
+                "can't read paddle version from [%s]" % CHECK_INFO_PB_FILENAME)
+            return False
+
+        if not self.check_info.hub_version:
+            logger.error(
+                "can't read hub version from [%s]" % CHECK_INFO_PB_FILENAME)
+            return False
+
+        if not self.check_info.module_proto_version:
+            logger.error("can't read module pb version from [%s]" %
+                         CHECK_INFO_PB_FILENAME)
+            return False
+
+        if not self.check_info.file_infos:
+            logger.error(
+                "can't read file info from [%s]" % CHECK_INFO_PB_FILENAME)
+            return False
+
+        return self.check_module() and self.check_compatibility()
+
+    def check_compatibility(self):
+        return self._check_module_proto_version() and self._check_hub_version(
+        ) and self._check_paddle_version()
+
+    def check_module(self):
+        return self._check_module_integrity() and self._check_dependency()
+
+    def _check_dependency(self):
+        return True
+
+    def _check_module_proto_version(self):
+        if self.module_proto_version != module_proto_version:
+            return False
+        return True
+
+    def _check_hub_version(self):
+        return True
+
+    def _check_paddle_version(self):
+        return True
+
+    def _check_module_integrity(self):
+
+        for file_info in self.file_infos:
+            file_type = file_info.type
+            file_path = file_info.file_name.replace(FILE_SEP, os.sep)
+            file_path = os.path.join(self.module_path, file_path)
+            if not os.path.exists(file_path):
+                if file_info.is_need:
+                    logger.error(
+                        "module lack of necessary file [%s]" % file_path)
+                    return False
+            else:
+                if file_type == check_info_pb2.FILE:
+                    if not os.path.isfile(file_path):
+                        logger.error("file type error %s" % file_path)
+                        return False
+
+                if file_type == check_info_pb2.DIR:
+                    if not os.path.isdir(file_path):
+                        logger.error("file type error %s" % file_path)
+                        return False
+        return True
+
+
+if __name__ == "__main__":
+    #     check_info = check_info_pb2.CheckInfo()
+    #     check_info.paddle_version = "1"
+    #     check_info.hub_version = "1"
+    #     check_info.module_proto_version = "1"
+    #     with open(os.path.join(".", CHECK_INFO_PB_FILENAME), "wb") as fi:
+    #         fi.write(check_info.SerializeToString())
+    check_info = ModuleChecker(
+        "/home/wuzewu/code/PaddleHub/demo/object-detection/hub_module_ssd")
+    print(check_info.check())

paddle_hub/module/module.py

@@ -20,7 +20,9 @@ from paddle_hub.tools.logger import logger
 from paddle_hub.tools.downloader import default_downloader
 from paddle_hub.tools import paddle_helper
 from paddle_hub.module import module_desc_pb2
+from paddle_hub.module import check_info_pb2
 from paddle_hub.module.signature import Signature, create_signature
+from paddle_hub.module.checker import ModuleChecker
 from paddle_hub.data.reader import yaml_reader
 from paddle_hub import version
 from paddle_hub.module.base_processor import BaseProcessor
@@ -156,6 +158,11 @@ class Module:
             self.assets.append(filepath)
 
     def _init_with_module_file(self, module_dir):
+        checker = ModuleChecker(module_dir)
+        if not checker.check():
+            logger.error("module check fail")
+            exit(1)
+
         self.helper = ModuleHelper(module_dir)
         with open(self.helper.module_desc_path(), "rb") as fi:
             self.desc.ParseFromString(fi.read())
@@ -464,7 +471,7 @@ class Module:
         # create module pb
         module_desc = module_desc_pb2.ModuleDesc()
         logger.info("hub version is %s" % version.hub_version)
-        logger.info("proto version is %s" % version.proto_version)
+        logger.info("module proto version is %s" % version.module_proto_version)
         logger.info("paddle version is %s" % paddle.__version__)
 
         feeded_var_names = [
@@ -530,3 +537,7 @@ class Module:
 
         # create assets
         self._dump_assets()
+
+        # create check info
+        checker = ModuleChecker(self.helper.module_dir)
+        checker.generate_check_info()

paddle_hub/version.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """ Store PaddleHub version string """
 hub_version = "0.1.0.dev"
-proto_version = "0.1.0"
+module_proto_version = "0.1.0"

scripts/gen_proto.sh

 #/bin/bash
 protoc -I=../paddle_hub/module --python_out=../paddle_hub/module ../paddle_hub/module/module_desc.proto
+protoc -I=../paddle_hub/module --python_out=../paddle_hub/module ../paddle_hub/module/check_info.proto