From 4d22e85b469d59de75514a2e46c965f4eba3f743 Mon Sep 17 00:00:00 2001 From: Megvii Engine Team Date: Fri, 10 Jun 2022 10:54:39 +0800 Subject: [PATCH] feat(ci): add completeness compatibility check GitOrigin-RevId: a7f75c4c5e9794c5ea201912c36cc6a7c95361b4 --- src/serialization/impl/opr_registry.cpp | 74 ++++++++++++++++++- src/serialization/impl/serializer_oss.cpp | 1 + src/serialization/impl/serializer_oss_v2.cpp | 1 + .../megbrain/serialization/opr_registry.h | 13 ++-- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/src/serialization/impl/opr_registry.cpp b/src/serialization/impl/opr_registry.cpp index 11d65a037..69ad8676b 100644 --- a/src/serialization/impl/opr_registry.cpp +++ b/src/serialization/impl/opr_registry.cpp @@ -25,6 +25,11 @@ struct StaticData { //! load/shallow copy and version_type_reg_map is used for Operator dump ThinHashMap> version_id_reg_map; ThinHashMap> version_type_reg_map; +#if MGB_ENABLE_DEBUG_UTIL + std::unordered_map> dumped_opr; + MGB_MUTEX g_record_map_mtx; + bool recorded = false; +#endif }; StaticData& static_data() { @@ -235,17 +240,78 @@ void OprRegistry::add_using_dynamic_loader( } #if MGB_ENABLE_DEBUG_UTIL -std::vector> OprRegistry::dump_registries() { +std::vector>> OprRegistry:: + dump_registries() { auto&& id2reg = static_data().id2reg; - std::vector> result; + std::vector>> result; + //! version 1 is old register, version 2 is registerV2 + result.resize(CURRENT_VERSION + 1); + std::vector> old_version; for (auto iter = id2reg.begin(); iter != id2reg.end(); ++iter) { if (iter->second.name.size() == 0) - result.push_back({iter->first, ""}); + old_version.push_back(std::make_pair(iter->first, "")); else - result.push_back({iter->first, iter->second.name}); + old_version.push_back(std::make_pair(iter->first, iter->second.name)); + } + result[VERSION_1] = old_version; + auto&& version_id_reg_map = static_data().version_id_reg_map; + for (int version_id = CURRENT_VERSION; version_id > 1; version_id--) { + std::vector> version_opr; + auto&& version_map = version_id_reg_map[version_id]; + for (auto&& it : version_map) { + if (it.second.name.size() == 0) + version_opr.push_back(std::make_pair(it.first, "")); + else + version_opr.push_back(std::make_pair(it.first, it.second.name)); + } + result[version_id] = version_opr; } return result; } + +std::vector>> OprRegistry:: + recorded_serialized_oprs(bool begin_record, bool end_record) { + MGB_LOCK_GUARD(static_data().g_record_map_mtx); + if (begin_record) { + static_data().recorded = true; + return {}; + } + if (end_record) { + static_data().recorded = false; + std::vector>> result; + result.resize(CURRENT_VERSION + 1); + auto& recorded = static_data().dumped_opr; + for (int version_id = CURRENT_VERSION; version_id > 0; version_id--) { + std::vector> version_opr; + auto&& version_map = recorded[version_id]; + for (auto&& it : version_map) { + if (it.second.size() == 0) + version_opr.push_back(std::make_pair(it.first, "")); + else + version_opr.push_back(std::make_pair(it.first, it.second)); + } + result[version_id] = version_opr; + } + static_data().dumped_opr.clear(); + return result; + } + return {}; +} + +void mgb::serialization::record_opr_dumped( + const size_t id, std::string name, int version) { + if (static_data().recorded) { + MGB_LOCK_GUARD(static_data().g_record_map_mtx); + auto& opr_dumped = static_data().dumped_opr; + if (name.size() == 0) + opr_dumped[version][id] = ""; + else + opr_dumped[version][id] = name; + } +} +#else + +void mgb::serialization::record_opr_dumped(const size_t, std::string, int) {} #endif namespace { diff --git a/src/serialization/impl/serializer_oss.cpp b/src/serialization/impl/serializer_oss.cpp index 39e0bda6e..6d3183f57 100644 --- a/src/serialization/impl/serializer_oss.cpp +++ b/src/serialization/impl/serializer_oss.cpp @@ -307,6 +307,7 @@ GraphDumper::DumpResult GraphDumperOSS::dump( init_oprs_to_dump(output_vars); std::vector> oprs; for (auto&& i : m_oprs_to_dump) { + record_opr_dumped(i.second->persist_type_id, i.second->name, 0); oprs.emplace_back(build_single_opr(i.first, i.second)); } auto fb_oprs = m_builder.CreateVector(oprs); diff --git a/src/serialization/impl/serializer_oss_v2.cpp b/src/serialization/impl/serializer_oss_v2.cpp index e9d49a1c9..f0c5a4a3c 100644 --- a/src/serialization/impl/serializer_oss_v2.cpp +++ b/src/serialization/impl/serializer_oss_v2.cpp @@ -394,6 +394,7 @@ GraphDumper::DumpResult GraphDumperOSSV2::dump( init_oprs_to_dump(new_output_vars); std::vector> oprs; for (auto&& i : m_oprs_to_dump) { + record_opr_dumped(i.second->type_id, i.second->name, i.second->version); oprs.emplace_back(build_single_opr(i.first, i.second)); } auto fb_oprs = m_builder.CreateVector(oprs); diff --git a/src/serialization/include/megbrain/serialization/opr_registry.h b/src/serialization/include/megbrain/serialization/opr_registry.h index 1ad8a9afe..c623e1bfc 100644 --- a/src/serialization/include/megbrain/serialization/opr_registry.h +++ b/src/serialization/include/megbrain/serialization/opr_registry.h @@ -22,6 +22,8 @@ public: cg::OperatorNodeBase* opr() { return m_opr; } }; +void record_opr_dumped(const size_t id, std::string name, int version); + //! dump opr internal params to OprDumpContext using OprDumper = thin_function; @@ -83,8 +85,11 @@ struct OprRegistry { #if MGB_ENABLE_DEBUG_UTIL //! dump registered oprs - MGE_WIN_DECLSPEC_FUC static std::vector> + MGE_WIN_DECLSPEC_FUC static std::vector>> dump_registries(); + //! record all dumped/loaded oprs (hash_id --> type) + MGE_WIN_DECLSPEC_FUC static std::vector>> + recorded_serialized_oprs(bool begin_record, bool end_record); #endif }; @@ -113,12 +118,6 @@ struct OprRegistryV2 { MGE_WIN_DECLSPEC_FUC static const OprRegistryV2* versioned_find_by_typeinfo( Typeinfo* type, uint8_t version); - -#if MGB_ENABLE_DEBUG_UTIL - //! dump registered oprs - MGE_WIN_DECLSPEC_FUC static std::vector> - dump_registries(); -#endif }; } // namespace serialization -- GitLab