Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 56291b21 编写于 作者: G groot
浏览文件
操作

use rocksdb column family 


Former-commit-id: 8cbc5d91e02e6bbc0eff1e329b4c76687d2c1f74
上级 354f68a9
隐藏空白更改
内联 并排
Showing with 101 addition and 51 deletion
cpp/conf/server_config.yaml
浏览文件 @ 56291b21
server_config:
address: 0.0.0.0
address: 127.0.0.1
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
server_mode: thread_pool #optional: simple, thread_pool
db_config:
db_path: /var/vecwise
db_path: /tmp/vecwise
db_backend_url: http://127.0.0.1
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128
......@@ -13,7 +13,7 @@ db_config:
log_config:
global:
format: "%datetime | %level | %logger | %msg"
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
enabled: true
to_file: true
to_standard_output: true
......@@ -21,22 +21,22 @@ log_config:
performance_tracking: false
max_log_file_size: 2097152 # throw log files away after 2mb
debug:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
enabled: true
warning:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
trace:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
verbose:
format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg"
to_file: false
to_standard_output: true
error:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
fatal:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
cache_config:
cpu_cache_capacity: 16 # unit: GB
......
cpp/conf/vecwise_engine_log.conf
浏览文件 @ 56291b21
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = true
......@@ -8,12 +8,12 @@
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
......@@ -21,7 +21,7 @@
## Error logs
* ERROR:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"
\ No newline at end of file
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"
\ No newline at end of file
cpp/src/server/VecIdMapper.cpp
浏览文件 @ 56291b21
......@@ -13,6 +13,8 @@
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
namespace zilliz {
namespace vecwise {
namespace server {
......@@ -36,26 +38,33 @@ SimpleIdMapper::~SimpleIdMapper() {
}
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) {
ids_[nid] = sid;
//not thread-safe
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping[nid] = sid;
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
//not thread-safe
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ID_MAPPING& mapping = id_groups_[group];
for(size_t i = 0; i < nid.size(); i++) {
ids_[nid[i]] = sid[i];
mapping[nid[i]] = sid[i];
}
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const {
auto iter = ids_.find(nid);
if(iter == ids_.end()) {
//not thread-safe
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
ID_MAPPING& mapping = id_groups_[group];
auto iter = mapping.find(nid);
if(iter == mapping.end()) {
return SERVER_INVALID_ARGUMENT;
}
......@@ -64,13 +73,16 @@ ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
//not thread-safe
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ID_MAPPING& mapping = id_groups_[group];
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
auto iter = ids_.find(nid[i]);
if(iter == ids_.end()) {
auto iter = mapping.find(nid[i]);
if(iter == mapping.end()) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
err = SERVER_INVALID_ARGUMENT;
......@@ -83,8 +95,10 @@ ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector
return err;
}
ServerError SimpleIdMapper::Delete(const std::string& nid) {
ids_.erase(nid);
//not thread-safe
ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping.erase(nid);
return SERVER_SUCCESS;
}
......@@ -104,12 +118,43 @@ RocksIdMapper::RocksIdMapper() {
options.create_if_missing = true;
options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128);
//load column families
std::vector<std::string> column_names;
rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
for(auto& column_name : column_names) {
rocksdb::ColumnFamilyDescriptor desc;
desc.name = column_name;
column_families.emplace_back(desc);
}
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_);
std::vector<rocksdb::ColumnFamilyHandle*> column_handles;
s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
db_ = nullptr;
}
try {
rocksdb::ColumnFamilyHandle *cf;
s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "222", &cf);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
std::vector<std::string> column_families;
s = db_->ListColumnFamilies(options, db_path, &column_families);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
} catch(std::exception& ex) {
std::cout << ex.what() << std::endl;
}
}
RocksIdMapper::~RocksIdMapper() {
if(db_) {
......@@ -118,7 +163,7 @@ RocksIdMapper::~RocksIdMapper() {
}
}
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) {
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
......@@ -134,7 +179,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
......@@ -150,7 +195,7 @@ ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::v
return err;
}
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const {
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
......@@ -165,7 +210,7 @@ ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ServerError err = SERVER_SUCCESS;
......@@ -185,7 +230,7 @@ ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<
return err;
}
ServerError RocksIdMapper::Delete(const std::string& nid) {
ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
......
cpp/src/server/VecIdMapper.h
浏览文件 @ 56291b21
......@@ -25,14 +25,14 @@ public:
virtual ~IVecIdMapper(){}
virtual ServerError Put(const std::string& nid, const std::string& sid) = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
virtual ServerError Get(const std::string& nid, std::string& sid) const = 0;
virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0;
//NOTE: the 'sid' will be cleared at begin of the function
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const = 0;
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const = 0;
virtual ServerError Delete(const std::string& nid) = 0;
virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0;
};
class SimpleIdMapper : public IVecIdMapper{
......@@ -40,16 +40,17 @@ public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid) override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
private:
std::unordered_map<std::string, std::string> ids_;
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
class RocksIdMapper : public IVecIdMapper{
......@@ -57,13 +58,13 @@ public:
RocksIdMapper();
~RocksIdMapper();
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid) override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
private:
rocksdb::DB* db_;
......
cpp/unittest/server/idmapper_test.cpp
浏览文件 @ 56291b21
......@@ -44,9 +44,13 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
nid.clear();
sid.clear();
const int64_t count = 1000000;
for(int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i+100000));
sid.push_back("val_" + std::to_string(i));
{
server::TimeRecorder rc("prepare id data");
for (int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i + 100000));
sid.push_back("val_" + std::to_string(i));
}
rc.Record("done!");
}
{
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或