From b17b56f309d2c2acfd5356337568bb9c82cfd2c4 Mon Sep 17 00:00:00 2001 From: Megvii Engine Team Date: Thu, 19 Aug 2021 13:33:00 +0800 Subject: [PATCH] fix(build): fix copy bara error GitOrigin-RevId: 6d68824821bbf40b99c2ca9ce849c3471873cd40 --- lite/CMakeLists.txt | 20 -- lite/src/global.cpp | 6 +- lite/src/mge/algo_cache/redis_cache.cpp | 243 ------------------------ lite/src/mge/algo_cache/redis_cache.h | 49 ----- lite/src/mge/common.cpp | 5 - lite/src/network.cpp | 13 -- lite/src/parse_info/default_parse.h | 5 - 7 files changed, 1 insertion(+), 340 deletions(-) delete mode 100644 lite/src/mge/algo_cache/redis_cache.cpp delete mode 100644 lite/src/mge/algo_cache/redis_cache.h diff --git a/lite/CMakeLists.txt b/lite/CMakeLists.txt index 4ede647de..921b34abf 100644 --- a/lite/CMakeLists.txt +++ b/lite/CMakeLists.txt @@ -33,15 +33,6 @@ endif() # It defines macros needed by lite configure_file(src/lite_build_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/genfiles/lite_build_config.h) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/genfiles/lite_build_config.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - -# begin config lite -if(LITE_BUILD_WITH_MGE AND LITE_WITH_CUDA AND NOT WIN32) - # FXIME third_party cpp redis do not support build with clang-cl - file(GLOB_RECURSE SOURCES_CPP_REDIS ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/sources/*.cpp) - list(APPEND SOURCES_LITE ${SOURCES_CPP_REDIS}) - file(GLOB_RECURSE SOURCES_TACOPIE ${PROJECT_SOURCE_DIR}/third_party/tacopie/sources/*.cpp) - list(APPEND SOURCES_LITE ${SOURCES_TACOPIE}) -endif() add_library(lite_static STATIC ${SOURCES_LITE}) add_dependencies(lite_static lite_fbs_generate) include_directories($) @@ -89,17 +80,6 @@ endif() if(ANDROID) target_link_libraries(lite_shared_whl PRIVATE log) endif() - - -if(LITE_BUILD_WITH_MGE AND LITE_WITH_CUDA AND NOT WIN32) - # FXIME third_party cpp redis do not support build with clang-cl - target_include_directories(lite_static PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes) - target_include_directories(lite_static PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes) - target_include_directories(lite_shared PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes) - target_include_directories(lite_shared PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes) - target_include_directories(lite_shared_whl PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes) - target_include_directories(lite_shared_whl PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes) -endif() set(LITE_VERSION_SCRIPT ${PROJECT_SOURCE_DIR}/lite/src/version_lite.ld CACHE INTERNAL "Path to linker version script") add_custom_target(_lite_version_ld SOURCES ${LITE_VERSION_SCRIPT}) if(NOT MSVC AND NOT WIN32) diff --git a/lite/src/global.cpp b/lite/src/global.cpp index 5235d17b0..cecf0e20c 100644 --- a/lite/src/global.cpp +++ b/lite/src/global.cpp @@ -24,15 +24,11 @@ #include "megbrain/comp_node.h" #include "megbrain/serialization/extern_c_opr.h" #include "megbrain/version.h" -#include "megcore_opencl.h" #include "mge/algo_cache/file_cache.h" #include "mge/common.h" #if MGB_ENABLE_TENSOR_RT #include "megbrain/tensorrt/tensorrt_engine_cache.h" #endif -#if LITE_WITH_CUDA -#include "mge/algo_cache/redis_cache.h" -#endif #endif #include @@ -183,7 +179,7 @@ void lite::set_persistent_cache(const std::string& cache_path, void lite::dump_persistent_cache(const std::string& cache_path) { LITE_LOCK_GUARD(cache_control.cache_mutex); LITE_ASSERT(cache_control.cache_type == "file", - "now cache type is redis, it can't be dumped."); + "now cache type not correct, it can't be dumped."); static_cast(mgb::PersistentCache::inst()) .dump_cache(cache_path.c_str()); } diff --git a/lite/src/mge/algo_cache/redis_cache.cpp b/lite/src/mge/algo_cache/redis_cache.cpp deleted file mode 100644 index dbec3a970..000000000 --- a/lite/src/mge/algo_cache/redis_cache.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/** - * \file lite/src/mge/algo_cache/redis_cache.cpp - * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") - * - * Copyright (c) 2014-2021 Megvii Inc. All rights reserved. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ - -#include "lite_build_config.h" - -#if !defined(WIN32) && LITE_BUILD_WITH_MGE && LITE_WITH_CUDA -#include "../../misc.h" -#include "redis_cache.h" - -#include -#include - -namespace { - -/* -** Translation Table as described in RFC1113 -*/ -static const char cb64[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -/* -** Translation Table to decode: -*https://github.com/dgiardini/imgcalkap/blob/master/base64.c -*/ -static const char cd64[] = - "|$$$}rstuvwxyz{$$$$$$$>?@ABCDEFGHIJKLMNOPQRSTUVW$$$$$$XYZ[\\]^_`" - "abcdefghijklmnopq"; - -/* -** encodeblock -** -** encode 3 8-bit binary bytes as 4 '6-bit' characters -*/ -void encodeblock(unsigned char in[3], unsigned char out[4], int len) { - out[0] = cb64[in[0] >> 2]; - out[1] = cb64[((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)]; - out[2] = (unsigned char)(len > 1 ? cb64[((in[1] & 0x0f) << 2) | - ((in[2] & 0xc0) >> 6)] - : '='); - out[3] = (unsigned char)(len > 2 ? cb64[in[2] & 0x3f] : '='); -} - -/* -** decodeblock -** -** decode 4 '6-bit' characters into 3 8-bit binary bytes -*/ -void decodeblock(unsigned char in[4], unsigned char out[3]) { - out[0] = (unsigned char)(in[0] << 2 | in[1] >> 4); - out[1] = (unsigned char)(in[1] << 4 | in[2] >> 2); - out[2] = (unsigned char)(((in[2] << 6) & 0xc0) | in[3]); -} - -/** - * Encode string to base64 string - * @param input - source string - * @param outdata - target base64 string - * @param linesize - max size of line - */ -void encode(const std::vector& input, - std::vector& outdata, int linesize = 76) { - outdata.clear(); - - unsigned char in[3], out[4]; - int i, len, blocksout = 0; - size_t j = 0; - - auto* indata = reinterpret_cast(input.data()); - unsigned int insize = input.size(); - - while (j <= insize) { - len = 0; - for (i = 0; i < 3; i++) { - in[i] = (unsigned char)indata[j]; - j++; - if (j <= insize) { - len++; - } else { - in[i] = 0; - } - } - if (len) { - encodeblock(in, out, len); - for (i = 0; i < 4; i++) { - outdata.push_back(out[i]); - } - blocksout++; - } - if (blocksout >= (linesize / 4) || (j == insize)) { - if (blocksout) { - outdata.push_back('\r'); - outdata.push_back('\n'); - } - blocksout = 0; - } - } -} - -/** - * Decode base64 string ot source - * @param input - base64 string - * @param outdata - source string - */ -void decode(const std::vector& input, - std::vector& outdata) { - outdata.clear(); - - unsigned char in[4], out[3], v; - int i, len; - size_t j = 0; - - auto* indata = reinterpret_cast(input.data()); - unsigned int insize = input.size(); - - while (j <= insize) { - for (len = 0, i = 0; i < 4 && (j <= insize); i++) { - v = 0; - while ((j <= insize) && v == 0) { - v = (unsigned char)indata[j++]; - v = (unsigned char)((v < 43 || v > 122) ? 0 : cd64[v - 43]); - if (v) { - v = (unsigned char)((v == '$') ? 0 : v - 61); - } - } - if (j <= insize) { - len++; - if (v) { - in[i] = (unsigned char)(v - 1); - } - } else { - in[i] = 0; - } - } - if (len) { - decodeblock(in, out); - for (i = 0; i < len - 1; i++) { - outdata.push_back(out[i]); - } - } - } -} - -/** - * Encode binary data to base64 buffer - * @param input - source data - * @param outdata - target base64 buffer - * @param linesize - */ -void encode(const std::string& input, std::string& outdata, int linesize = 76) { - std::vector out; - std::vector in(input.begin(), input.end()); - encode(in, out, linesize); - outdata = std::string(out.begin(), out.end()); -} - -/** - * Decode base64 buffer to source binary data - * @param input - base64 buffer - * @param outdata - source binary data - */ -void decode(const std::string& input, std::string& outdata) { - std::vector in(input.begin(), input.end()); - std::vector out; - decode(in, out); - outdata = std::string(out.begin(), out.end()); -} - -} // namespace - -using namespace lite; - -RedisCache::RedisCache(std::string redis_ip, size_t port, std::string password) - : m_ip(redis_ip), m_port(port), m_password(password) { - m_client.auth(password); - m_client.connect( - m_ip, m_port, - [](const std::string& host, std::size_t port, - cpp_redis::connect_state status) { - if (status == cpp_redis::connect_state::dropped) { - LITE_LOG("client disconnected from %s.", host.c_str()); - LITE_LOG("Redis server connect to %s :%zu failed.", - host.c_str(), port); - } - }, - std::uint32_t(200)); -} - -mgb::Maybe RedisCache::get( - const std::string& category, const mgb::PersistentCache::Blob& key) { - LITE_LOCK_GUARD(m_mtx); - if (m_old == nullptr) { - return mgb::None; - } - auto mem_result = m_old->get(category, key); - if (mem_result.valid()) - return mem_result; - - std::string key_str(static_cast(key.ptr), key.size); - std::string redis_key_str; - encode(category + '@' + key_str, redis_key_str, 24); - auto result = m_client.get(redis_key_str); - m_client.sync_commit(std::chrono::milliseconds(100)); - LITE_ASSERT(is_valid()); - auto content = result.get(); - if (content.is_null()) - return mgb::None; - std::string decode_content; - decode(content.as_string(), decode_content); - m_old->put(category, key, {decode_content.data(), decode_content.length()}); - - return m_old->get(category, key); -} - -void RedisCache::put(const std::string& category, const Blob& key, - const mgb::PersistentCache::Blob& value) { - // ScopedTimer t1(std::string("put") + category); - LITE_LOCK_GUARD(m_mtx); - std::string key_str(static_cast(key.ptr), key.size); - std::string redis_key_str; - encode(category + '@' + key_str, redis_key_str); - std::string value_str(static_cast(value.ptr), value.size); - std::string redis_value_str; - encode(value_str, redis_value_str); - - auto result = m_client.set(redis_key_str, redis_value_str); - if (m_old == nullptr) { - return; - } - m_old->put(category, key, value); - m_client.sync_commit(std::chrono::milliseconds(100)); - LITE_ASSERT(is_valid()); -} -#endif - -// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}} diff --git a/lite/src/mge/algo_cache/redis_cache.h b/lite/src/mge/algo_cache/redis_cache.h deleted file mode 100644 index 7193a5fb8..000000000 --- a/lite/src/mge/algo_cache/redis_cache.h +++ /dev/null @@ -1,49 +0,0 @@ -/** - * \file lite/src/mge/algo_cache/redis_cache.h - * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") - * - * Copyright (c) 2014-2021 Megvii Inc. All rights reserved. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ - -#pragma once - -#include "lite_build_config.h" - -#if !defined(WIN32) && LITE_BUILD_WITH_MGE && LITE_WITH_CUDA -#include -#include -#include -#include "megbrain/utils/persistent_cache.h" - -namespace lite { - -//! TODO: fix one thread set cache when other threads is using old cache -class RedisCache final : public mgb::PersistentCache { -public: - RedisCache(std::string redis_ip, size_t port, std::string password); - - bool is_valid() { return m_client.is_connected(); } - ~RedisCache() {} - void init(std::shared_ptr old) { m_old = old; } - - mgb::Maybe get(const std::string& category, const Blob& key) override; - - void put(const std::string& category, const Blob& key, - const Blob& value) override; - -private: - std::shared_ptr m_old; - LITE_MUTEX m_mtx; - cpp_redis::client m_client; - const std::string m_ip; - const size_t m_port; - const std::string m_password; -}; - -} // namespace lite -#endif -// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}} diff --git a/lite/src/mge/common.cpp b/lite/src/mge/common.cpp index 12e9ad1aa..b4740de58 100644 --- a/lite/src/mge/common.cpp +++ b/lite/src/mge/common.cpp @@ -154,9 +154,6 @@ mgb::CompNode::Locator lite::to_compnode_locator(const LiteDeviceType& device) { case LiteDeviceType::LITE_ATLAS: loc.type = mgb::CompNode::DeviceType::ATLAS; break; - case LiteDeviceType::LITE_OPENCL: - loc.type = mgb::CompNode::DeviceType::OPENCL; - break; case LiteDeviceType::LITE_DEVICE_DEFAULT: loc.type = mgb::CompNode::DeviceType::UNSPEC; break; @@ -178,8 +175,6 @@ LiteDeviceType lite::get_device_from_locator( return LiteDeviceType::LITE_CUDA; case mgb::CompNode::DeviceType::ATLAS: return LiteDeviceType::LITE_ATLAS; - case mgb::CompNode::DeviceType::OPENCL: - return LiteDeviceType::LITE_OPENCL; case mgb::CompNode::DeviceType::UNSPEC: return LiteDeviceType::LITE_DEVICE_DEFAULT; default: diff --git a/lite/src/network.cpp b/lite/src/network.cpp index 7874c0dcd..42b937e9c 100644 --- a/lite/src/network.cpp +++ b/lite/src/network.cpp @@ -40,10 +40,6 @@ Network::Network(const Config& config, const NetworkIO& network_io) { m_impl = call_func>( "create_network"); - } else if (config.backend == LiteBackend::LITE_RK_NPU) { - m_impl = call_func>( - "create_network"); } m_impl->set_config(config); m_impl->set_io(network_io); @@ -58,10 +54,6 @@ Network::Network(const NetworkIO& network_io, const Config& config) { m_impl = call_func>( "create_network"); - } else if (config.backend == LiteBackend::LITE_RK_NPU) { - m_impl = call_func>( - "create_network"); } m_impl->set_config(config); m_impl->set_io(network_io); @@ -106,11 +98,6 @@ void Network::prase_model(std::shared_ptr model_data, size_t size) { m_impl.reset(try_call_func( "parse_model")); - } else if (m_config.backend == LiteBackend::LITE_RK_NPU && - m_impl->get_backend_type() != LiteBackend::LITE_RK_NPU) { - m_impl.reset(try_call_func( - "parse_model")); } m_impl->set_config(m_config); m_impl->set_io(m_network_io); diff --git a/lite/src/parse_info/default_parse.h b/lite/src/parse_info/default_parse.h index a2f7a3ff4..dab9d6a37 100644 --- a/lite/src/parse_info/default_parse.h +++ b/lite/src/parse_info/default_parse.h @@ -63,9 +63,6 @@ bool default_parse_info( if (info["backend"] == "MGE") { config.backend = LiteBackend::LITE_DEFAULT; } - if (info["backend"] == "RK") { - config.backend = LiteBackend::LITE_RK_NPU; - } } auto get_device_type = [](std::string type) -> LiteDeviceType { @@ -73,8 +70,6 @@ bool default_parse_info( return LiteDeviceType::LITE_CPU; if (type == "CUDA") return LiteDeviceType::LITE_CUDA; - if (type == "OPENCL") - return LiteDeviceType::LITE_OPENCL; if (type == "ATLAS") return LiteDeviceType::LITE_ATLAS; if (type == "NPU") -- GitLab