Skip to content

MegEngine
MegEngine

sadsadfsdsfdsfd / MegEngine
与 Fork 源项目一致

Fork自 MegEngine 天元 / MegEngine

通知 1
Star 0
Fork 0
MegEngine
MegEngine

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 b17b56f3 编写于 作者: M Megvii Engine Team
浏览文件
操作

fix(build): fix copy bara error 

GitOrigin-RevId: 6d68824821bbf40b99c2ca9ce849c3471873cd40
上级 3c6665f7
隐藏空白更改
内联 并排
Showing with 1 addition and 340 deletion
lite/CMakeLists.txt
浏览文件 @ b17b56f3
......@@ -33,15 +33,6 @@ endif()
# It defines macros needed by lite
configure_file(src/lite_build_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/genfiles/lite_build_config.h)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/genfiles/lite_build_config.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
# begin config lite
if(LITE_BUILD_WITH_MGE AND LITE_WITH_CUDA AND NOT WIN32)
# FXIME third_party cpp redis do not support build with clang-cl
file(GLOB_RECURSE SOURCES_CPP_REDIS ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/sources/*.cpp)
list(APPEND SOURCES_LITE ${SOURCES_CPP_REDIS})
file(GLOB_RECURSE SOURCES_TACOPIE ${PROJECT_SOURCE_DIR}/third_party/tacopie/sources/*.cpp)
list(APPEND SOURCES_LITE ${SOURCES_TACOPIE})
endif()
add_library(lite_static STATIC ${SOURCES_LITE})
add_dependencies(lite_static lite_fbs_generate)
include_directories($<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/genfiles>)
......@@ -89,17 +80,6 @@ endif()
if(ANDROID)
target_link_libraries(lite_shared_whl PRIVATE log)
endif()
if(LITE_BUILD_WITH_MGE AND LITE_WITH_CUDA AND NOT WIN32)
# FXIME third_party cpp redis do not support build with clang-cl
target_include_directories(lite_static PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes)
target_include_directories(lite_static PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes)
target_include_directories(lite_shared PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes)
target_include_directories(lite_shared PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes)
target_include_directories(lite_shared_whl PRIVATE ${PROJECT_SOURCE_DIR}/third_party/cpp_redis/includes)
target_include_directories(lite_shared_whl PRIVATE ${PROJECT_SOURCE_DIR}/third_party/tacopie/includes)
endif()
set(LITE_VERSION_SCRIPT ${PROJECT_SOURCE_DIR}/lite/src/version_lite.ld CACHE INTERNAL "Path to linker version script")
add_custom_target(_lite_version_ld SOURCES ${LITE_VERSION_SCRIPT})
if(NOT MSVC AND NOT WIN32)
......
lite/src/global.cpp
浏览文件 @ b17b56f3
......@@ -24,15 +24,11 @@
#include "megbrain/comp_node.h"
#include "megbrain/serialization/extern_c_opr.h"
#include "megbrain/version.h"
#include "megcore_opencl.h"
#include "mge/algo_cache/file_cache.h"
#include "mge/common.h"
#if MGB_ENABLE_TENSOR_RT
#include "megbrain/tensorrt/tensorrt_engine_cache.h"
#endif
#if LITE_WITH_CUDA
#include "mge/algo_cache/redis_cache.h"
#endif
#endif
#include <mutex>
......@@ -183,7 +179,7 @@ void lite::set_persistent_cache(const std::string& cache_path,
void lite::dump_persistent_cache(const std::string& cache_path) {
LITE_LOCK_GUARD(cache_control.cache_mutex);
LITE_ASSERT(cache_control.cache_type == "file",
"now cache type is redis, it can't be dumped.");
"now cache type not correct, it can't be dumped.");
static_cast<InFilePersistentCache&>(mgb::PersistentCache::inst())
.dump_cache(cache_path.c_str());
}
......
lite/src/mge/algo_cache/redis_cache.cpp 已删除 100644 → 0
浏览文件 @ 3c6665f7
/**
* \file lite/src/mge/algo_cache/redis_cache.cpp
* MegEngine is Licensed under the Apache License, Version 2.0 (the "License")
*
* Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
#include "lite_build_config.h"
#if !defined(WIN32) && LITE_BUILD_WITH_MGE && LITE_WITH_CUDA
#include "../../misc.h"
#include "redis_cache.h"
#include <iostream>
#include <vector>
namespace {
/*
** Translation Table as described in RFC1113
*/
static const char cb64[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
/*
** Translation Table to decode:
*https://github.com/dgiardini/imgcalkap/blob/master/base64.c
*/
static const char cd64[] =
"|$$$}rstuvwxyz{$$$$$$$>?@ABCDEFGHIJKLMNOPQRSTUVW$$$$$$XYZ[\\]^_`"
"abcdefghijklmnopq";
/*
** encodeblock
**
** encode 3 8-bit binary bytes as 4 '6-bit' characters
*/
void encodeblock(unsigned char in[3], unsigned char out[4], int len) {
out[0] = cb64[in[0] >> 2];
out[1] = cb64[((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)];
out[2] = (unsigned char)(len > 1 ? cb64[((in[1] & 0x0f) << 2) |
((in[2] & 0xc0) >> 6)]
: '=');
out[3] = (unsigned char)(len > 2 ? cb64[in[2] & 0x3f] : '=');
}
/*
** decodeblock
**
** decode 4 '6-bit' characters into 3 8-bit binary bytes
*/
void decodeblock(unsigned char in[4], unsigned char out[3]) {
out[0] = (unsigned char)(in[0] << 2 | in[1] >> 4);
out[1] = (unsigned char)(in[1] << 4 | in[2] >> 2);
out[2] = (unsigned char)(((in[2] << 6) & 0xc0) | in[3]);
}
/**
* Encode string to base64 string
* @param input - source string
* @param outdata - target base64 string
* @param linesize - max size of line
*/
void encode(const std::vector<std::uint8_t>& input,
std::vector<std::uint8_t>& outdata, int linesize = 76) {
outdata.clear();
unsigned char in[3], out[4];
int i, len, blocksout = 0;
size_t j = 0;
auto* indata = reinterpret_cast<const unsigned char*>(input.data());
unsigned int insize = input.size();
while (j <= insize) {
len = 0;
for (i = 0; i < 3; i++) {
in[i] = (unsigned char)indata[j];
j++;
if (j <= insize) {
len++;
} else {
in[i] = 0;
}
}
if (len) {
encodeblock(in, out, len);
for (i = 0; i < 4; i++) {
outdata.push_back(out[i]);
}
blocksout++;
}
if (blocksout >= (linesize / 4) || (j == insize)) {
if (blocksout) {
outdata.push_back('\r');
outdata.push_back('\n');
}
blocksout = 0;
}
}
}
/**
* Decode base64 string ot source
* @param input - base64 string
* @param outdata - source string
*/
void decode(const std::vector<std::uint8_t>& input,
std::vector<std::uint8_t>& outdata) {
outdata.clear();
unsigned char in[4], out[3], v;
int i, len;
size_t j = 0;
auto* indata = reinterpret_cast<const unsigned char*>(input.data());
unsigned int insize = input.size();
while (j <= insize) {
for (len = 0, i = 0; i < 4 && (j <= insize); i++) {
v = 0;
while ((j <= insize) && v == 0) {
v = (unsigned char)indata[j++];
v = (unsigned char)((v < 43 || v > 122) ? 0 : cd64[v - 43]);
if (v) {
v = (unsigned char)((v == '$') ? 0 : v - 61);
}
}
if (j <= insize) {
len++;
if (v) {
in[i] = (unsigned char)(v - 1);
}
} else {
in[i] = 0;
}
}
if (len) {
decodeblock(in, out);
for (i = 0; i < len - 1; i++) {
outdata.push_back(out[i]);
}
}
}
}
/**
* Encode binary data to base64 buffer
* @param input - source data
* @param outdata - target base64 buffer
* @param linesize
*/
void encode(const std::string& input, std::string& outdata, int linesize = 76) {
std::vector<std::uint8_t> out;
std::vector<std::uint8_t> in(input.begin(), input.end());
encode(in, out, linesize);
outdata = std::string(out.begin(), out.end());
}
/**
* Decode base64 buffer to source binary data
* @param input - base64 buffer
* @param outdata - source binary data
*/
void decode(const std::string& input, std::string& outdata) {
std::vector<std::uint8_t> in(input.begin(), input.end());
std::vector<std::uint8_t> out;
decode(in, out);
outdata = std::string(out.begin(), out.end());
}
} // namespace
using namespace lite;
RedisCache::RedisCache(std::string redis_ip, size_t port, std::string password)
: m_ip(redis_ip), m_port(port), m_password(password) {
m_client.auth(password);
m_client.connect(
m_ip, m_port,
[](const std::string& host, std::size_t port,
cpp_redis::connect_state status) {
if (status == cpp_redis::connect_state::dropped) {
LITE_LOG("client disconnected from %s.", host.c_str());
LITE_LOG("Redis server connect to %s :%zu failed.",
host.c_str(), port);
}
},
std::uint32_t(200));
}
mgb::Maybe<mgb::PersistentCache::Blob> RedisCache::get(
const std::string& category, const mgb::PersistentCache::Blob& key) {
LITE_LOCK_GUARD(m_mtx);
if (m_old == nullptr) {
return mgb::None;
}
auto mem_result = m_old->get(category, key);
if (mem_result.valid())
return mem_result;
std::string key_str(static_cast<const char*>(key.ptr), key.size);
std::string redis_key_str;
encode(category + '@' + key_str, redis_key_str, 24);
auto result = m_client.get(redis_key_str);
m_client.sync_commit<double, std::milli>(std::chrono::milliseconds(100));
LITE_ASSERT(is_valid());
auto content = result.get();
if (content.is_null())
return mgb::None;
std::string decode_content;
decode(content.as_string(), decode_content);
m_old->put(category, key, {decode_content.data(), decode_content.length()});
return m_old->get(category, key);
}
void RedisCache::put(const std::string& category, const Blob& key,
const mgb::PersistentCache::Blob& value) {
// ScopedTimer t1(std::string("put") + category);
LITE_LOCK_GUARD(m_mtx);
std::string key_str(static_cast<const char*>(key.ptr), key.size);
std::string redis_key_str;
encode(category + '@' + key_str, redis_key_str);
std::string value_str(static_cast<const char*>(value.ptr), value.size);
std::string redis_value_str;
encode(value_str, redis_value_str);
auto result = m_client.set(redis_key_str, redis_value_str);
if (m_old == nullptr) {
return;
}
m_old->put(category, key, value);
m_client.sync_commit<double, std::milli>(std::chrono::milliseconds(100));
LITE_ASSERT(is_valid());
}
#endif
// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}}
lite/src/mge/algo_cache/redis_cache.h 已删除 100644 → 0
浏览文件 @ 3c6665f7
/**
* \file lite/src/mge/algo_cache/redis_cache.h
* MegEngine is Licensed under the Apache License, Version 2.0 (the "License")
*
* Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
#pragma once
#include "lite_build_config.h"
#if !defined(WIN32) && LITE_BUILD_WITH_MGE && LITE_WITH_CUDA
#include <cpp_redis/cpp_redis>
#include <string>
#include <vector>
#include "megbrain/utils/persistent_cache.h"
namespace lite {
//! TODO: fix one thread set cache when other threads is using old cache
class RedisCache final : public mgb::PersistentCache {
public:
RedisCache(std::string redis_ip, size_t port, std::string password);
bool is_valid() { return m_client.is_connected(); }
~RedisCache() {}
void init(std::shared_ptr<mgb::PersistentCache> old) { m_old = old; }
mgb::Maybe<Blob> get(const std::string& category, const Blob& key) override;
void put(const std::string& category, const Blob& key,
const Blob& value) override;
private:
std::shared_ptr<mgb::PersistentCache> m_old;
LITE_MUTEX m_mtx;
cpp_redis::client m_client;
const std::string m_ip;
const size_t m_port;
const std::string m_password;
};
} // namespace lite
#endif
// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}}
lite/src/mge/common.cpp
浏览文件 @ b17b56f3
......@@ -154,9 +154,6 @@ mgb::CompNode::Locator lite::to_compnode_locator(const LiteDeviceType& device) {
case LiteDeviceType::LITE_ATLAS:
loc.type = mgb::CompNode::DeviceType::ATLAS;
break;
case LiteDeviceType::LITE_OPENCL:
loc.type = mgb::CompNode::DeviceType::OPENCL;
break;
case LiteDeviceType::LITE_DEVICE_DEFAULT:
loc.type = mgb::CompNode::DeviceType::UNSPEC;
break;
......@@ -178,8 +175,6 @@ LiteDeviceType lite::get_device_from_locator(
return LiteDeviceType::LITE_CUDA;
case mgb::CompNode::DeviceType::ATLAS:
return LiteDeviceType::LITE_ATLAS;
case mgb::CompNode::DeviceType::OPENCL:
return LiteDeviceType::LITE_OPENCL;
case mgb::CompNode::DeviceType::UNSPEC:
return LiteDeviceType::LITE_DEVICE_DEFAULT;
default:
......
lite/src/network.cpp
浏览文件 @ b17b56f3
......@@ -40,10 +40,6 @@ Network::Network(const Config& config, const NetworkIO& network_io) {
m_impl = call_func<NetworkImplDft,
std::unique_ptr<lite::Network::NetworkImplBase>>(
"create_network");
} else if (config.backend == LiteBackend::LITE_RK_NPU) {
m_impl = call_func<NetworkImplRK,
std::unique_ptr<lite::Network::NetworkImplBase>>(
"create_network");
}
m_impl->set_config(config);
m_impl->set_io(network_io);
......@@ -58,10 +54,6 @@ Network::Network(const NetworkIO& network_io, const Config& config) {
m_impl = call_func<NetworkImplDft,
std::unique_ptr<lite::Network::NetworkImplBase>>(
"create_network");
} else if (config.backend == LiteBackend::LITE_RK_NPU) {
m_impl = call_func<NetworkImplRK,
std::unique_ptr<lite::Network::NetworkImplBase>>(
"create_network");
}
m_impl->set_config(config);
m_impl->set_io(network_io);
......@@ -106,11 +98,6 @@ void Network::prase_model(std::shared_ptr<void> model_data, size_t size) {
m_impl.reset(try_call_func<NetworkImplDft,
lite::Network::NetworkImplBase*>(
"parse_model"));
} else if (m_config.backend == LiteBackend::LITE_RK_NPU &&
m_impl->get_backend_type() != LiteBackend::LITE_RK_NPU) {
m_impl.reset(try_call_func<NetworkImplRK,
lite::Network::NetworkImplBase*>(
"parse_model"));
}
m_impl->set_config(m_config);
m_impl->set_io(m_network_io);
......
lite/src/parse_info/default_parse.h
浏览文件 @ b17b56f3
......@@ -63,9 +63,6 @@ bool default_parse_info(
if (info["backend"] == "MGE") {
config.backend = LiteBackend::LITE_DEFAULT;
}
if (info["backend"] == "RK") {
config.backend = LiteBackend::LITE_RK_NPU;
}
}
auto get_device_type = [](std::string type) -> LiteDeviceType {
......@@ -73,8 +70,6 @@ bool default_parse_info(
return LiteDeviceType::LITE_CPU;
if (type == "CUDA")
return LiteDeviceType::LITE_CUDA;
if (type == "OPENCL")
return LiteDeviceType::LITE_OPENCL;
if (type == "ATLAS")
return LiteDeviceType::LITE_ATLAS;
if (type == "NPU")
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或