Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 35cfea92 编写于 作者: Y Yu Kun
浏览文件
操作

Merge remote-tracking branch 'upstream/branch-0.5.0' into branch-0.5.0-yk 


Former-commit-id: dd49cc79b5e934e8b2daee1e926df2731655fd4a
上级 165296c2 464a3415
展开全部 隐藏空白更改
内联 并排
Showing with 4494 addition and 4411 deletion
.clang-format
浏览文件 @ 35cfea92
......@@ -18,3 +18,8 @@
BasedOnStyle: Google
DerivePointerAlignment: false
ColumnLimit: 120
IndentWidth: 4
AccessModifierOffset: -3
AlwaysBreakAfterReturnType: All
AllowShortBlocksOnASingleLine: false
AllowShortFunctionsOnASingleLine: false
cpp/build-support/code_style_clion.xml 0 → 100644
浏览文件 @ 35cfea92
<code_scheme name="Default" version="173">
<Objective-C>
<option name="INDENT_NAMESPACE_MEMBERS" value="0" />
<option name="INDENT_VISIBILITY_KEYWORDS" value="1" />
<option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true" />
<option name="KEEP_CASE_EXPRESSIONS_IN_ONE_LINE" value="true" />
<option name="FUNCTION_NON_TOP_AFTER_RETURN_TYPE_WRAP" value="0" />
<option name="FUNCTION_TOP_AFTER_RETURN_TYPE_WRAP" value="2" />
<option name="FUNCTION_PARAMETERS_WRAP" value="5" />
<option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true" />
<option name="CLASS_CONSTRUCTOR_INIT_LIST_WRAP" value="5" />
<option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false" />
<option name="SPACE_BEFORE_PROTOCOLS_BRACKETS" value="false" />
<option name="SPACE_BEFORE_POINTER_IN_DECLARATION" value="false" />
<option name="SPACE_AFTER_POINTER_IN_DECLARATION" value="true" />
<option name="SPACE_BEFORE_REFERENCE_IN_DECLARATION" value="false" />
<option name="SPACE_AFTER_REFERENCE_IN_DECLARATION" value="true" />
<option name="KEEP_BLANK_LINES_BEFORE_END" value="1" />
</Objective-C>
<codeStyleSettings language="ObjectiveC">
<option name="KEEP_BLANK_LINES_IN_DECLARATIONS" value="1" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
<option name="BLANK_LINES_AROUND_CLASS" value="0" />
<option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0" />
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
<option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false" />
<option name="SPACE_AFTER_TYPE_CAST" value="false" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="KEEP_SIMPLE_BLOCKS_IN_ONE_LINE" value="false" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ASSIGNMENT_WRAP" value="1" />
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
</indentOptions>
</codeStyleSettings>
</code_scheme>
\ No newline at end of file
cpp/build.sh
浏览文件 @ 35cfea92
......@@ -104,13 +104,13 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
fi
echo "cpplint check passed!"
# # clang-format check
# make check-clang-format
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-format check failed"
# exit 1
# fi
# echo "clang-format check passed!"
# clang-format check
make check-clang-format
if [ $? -ne 0 ]; then
echo "ERROR! clang-format check failed"
exit 1
fi
echo "clang-format check passed!"
#
# # clang-tidy check
# make check-clang-tidy
......
cpp/src/cache/Cache.h
浏览文件 @ 35cfea92
......@@ -15,55 +15,67 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "LRU.h"
#include "utils/Log.h"
#include <string>
#include <mutex>
#include <atomic>
#include <mutex>
#include <set>
#include <string>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
class Cache {
public:
//mem_capacity, units:GB
// mem_capacity, units:GB
Cache(int64_t capacity_gb, uint64_t cache_max_count);
~Cache() = default;
int64_t usage() const {
int64_t
usage() const {
return usage_;
}
int64_t capacity() const {
int64_t
capacity() const {
return capacity_;
} //unit: BYTE
void set_capacity(int64_t capacity); //unit: BYTE
} // unit: BYTE
void
set_capacity(int64_t capacity); // unit: BYTE
double freemem_percent() const {
double
freemem_percent() const {
return freemem_percent_;
}
void set_freemem_percent(double percent) {
void
set_freemem_percent(double percent) {
freemem_percent_ = percent;
}
size_t size() const;
bool exists(const std::string &key);
ItemObj get(const std::string &key);
void insert(const std::string &key, const ItemObj &item);
void erase(const std::string &key);
void print();
void clear();
size_t
size() const;
bool
exists(const std::string& key);
ItemObj
get(const std::string& key);
void
insert(const std::string& key, const ItemObj& item);
void
erase(const std::string& key);
void
print();
void
clear();
private:
void free_memory();
void
free_memory();
private:
int64_t usage_;
......@@ -74,8 +86,8 @@ class Cache {
mutable std::mutex mutex_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
#include "cache/Cache.inl"
cpp/src/cache/CacheMgr.h
浏览文件 @ 35cfea92
......@@ -15,40 +15,49 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "Cache.h"
#include "utils/Log.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
class CacheMgr {
public:
virtual uint64_t ItemCount() const;
virtual uint64_t
ItemCount() const;
virtual bool ItemExists(const std::string &key);
virtual bool
ItemExists(const std::string& key);
virtual ItemObj GetItem(const std::string &key);
virtual ItemObj
GetItem(const std::string& key);
virtual void InsertItem(const std::string &key, const ItemObj &data);
virtual void
InsertItem(const std::string& key, const ItemObj& data);
virtual void EraseItem(const std::string &key);
virtual void
EraseItem(const std::string& key);
virtual void PrintInfo();
virtual void
PrintInfo();
virtual void ClearCache();
virtual void
ClearCache();
int64_t CacheUsage() const;
int64_t CacheCapacity() const;
void SetCapacity(int64_t capacity);
int64_t
CacheUsage() const;
int64_t
CacheCapacity() const;
void
SetCapacity(int64_t capacity);
protected:
CacheMgr();
......@@ -59,8 +68,8 @@ class CacheMgr {
CachePtr cache_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
#include "cache/CacheMgr.inl"
cpp/src/cache/CpuCacheMgr.cpp
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "utils/Log.h"
......@@ -31,7 +30,7 @@ constexpr int64_t unit = 1024 * 1024 * 1024;
}
CpuCacheMgr::CpuCacheMgr() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s;
int32_t cpu_cache_cap;
......@@ -50,19 +49,19 @@ CpuCacheMgr::CpuCacheMgr() {
if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) {
cache_->set_freemem_percent(cpu_cache_threshold);
} else {
SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold
<< ", by default set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to "
<< cache_->freemem_percent();
}
}
CpuCacheMgr *
CpuCacheMgr*
CpuCacheMgr::GetInstance() {
static CpuCacheMgr s_mgr;
return &s_mgr;
}
engine::VecIndexPtr
CpuCacheMgr::GetIndex(const std::string &key) {
CpuCacheMgr::GetIndex(const std::string& key) {
DataObjPtr obj = GetItem(key);
if (obj != nullptr) {
return obj->data();
......@@ -71,6 +70,6 @@ CpuCacheMgr::GetIndex(const std::string &key) {
return nullptr;
}
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/cache/CpuCacheMgr.h
浏览文件 @ 35cfea92
......@@ -20,8 +20,8 @@
#include "CacheMgr.h"
#include "DataObj.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -32,12 +32,14 @@ class CpuCacheMgr : public CacheMgr<DataObjPtr> {
CpuCacheMgr();
public:
//TODO: use smart pointer instead
static CpuCacheMgr *GetInstance();
// TODO: use smart pointer instead
static CpuCacheMgr*
GetInstance();
engine::VecIndexPtr GetIndex(const std::string &key);
engine::VecIndexPtr
GetIndex(const std::string& key);
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/cache/DataObj.h
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "src/wrapper/VecIndex.h"
......@@ -28,24 +27,24 @@ namespace cache {
class DataObj {
public:
explicit DataObj(const engine::VecIndexPtr &index)
: index_(index) {
explicit DataObj(const engine::VecIndexPtr& index) : index_(index) {
}
DataObj(const engine::VecIndexPtr &index, int64_t size)
: index_(index),
size_(size) {
DataObj(const engine::VecIndexPtr& index, int64_t size) : index_(index), size_(size) {
}
engine::VecIndexPtr data() {
engine::VecIndexPtr
data() {
return index_;
}
const engine::VecIndexPtr &data() const {
const engine::VecIndexPtr&
data() const {
return index_;
}
int64_t size() const {
int64_t
size() const {
if (index_ == nullptr) {
return 0;
}
......@@ -64,6 +63,6 @@ class DataObj {
using DataObjPtr = std::shared_ptr<DataObj>;
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/cache/GpuCacheMgr.cpp
浏览文件 @ 35cfea92
......@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/GpuCacheMgr.h"
#include "utils/Log.h"
#include "server/Config.h"
#include "utils/Log.h"
#include <sstream>
#include <utility>
......@@ -35,7 +34,7 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
}
GpuCacheMgr::GpuCacheMgr() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s;
int32_t gpu_cache_cap;
......@@ -54,12 +53,12 @@ GpuCacheMgr::GpuCacheMgr() {
if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(gpu_mem_threshold);
} else {
SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold
<< ", by default set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold << ", by default set to "
<< cache_->freemem_percent();
}
}
GpuCacheMgr *
GpuCacheMgr*
GpuCacheMgr::GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
......@@ -74,7 +73,7 @@ GpuCacheMgr::GetInstance(uint64_t gpu_id) {
}
engine::VecIndexPtr
GpuCacheMgr::GetIndex(const std::string &key) {
GpuCacheMgr::GetIndex(const std::string& key) {
DataObjPtr obj = GetItem(key);
if (obj != nullptr) {
return obj->data();
......@@ -83,6 +82,6 @@ GpuCacheMgr::GetIndex(const std::string &key) {
return nullptr;
}
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/cache/GpuCacheMgr.h
浏览文件 @ 35cfea92
......@@ -15,13 +15,12 @@
// specific language governing permissions and limitations
// under the License.
#include "CacheMgr.h"
#include "DataObj.h"
#include <unordered_map>
#include <memory>
#include <string>
#include <unordered_map>
namespace zilliz {
namespace milvus {
......@@ -34,15 +33,17 @@ class GpuCacheMgr : public CacheMgr<DataObjPtr> {
public:
GpuCacheMgr();
static GpuCacheMgr *GetInstance(uint64_t gpu_id);
static GpuCacheMgr*
GetInstance(uint64_t gpu_id);
engine::VecIndexPtr GetIndex(const std::string &key);
engine::VecIndexPtr
GetIndex(const std::string& key);
private:
static std::mutex mutex_;
static std::unordered_map<uint64_t, GpuCacheMgrPtr> instance_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/cache/LRU.h
浏览文件 @ 35cfea92
......@@ -15,20 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <unordered_map>
#include <list>
#include <cstddef>
#include <list>
#include <stdexcept>
#include <unordered_map>
#include <utility>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename key_t, typename value_t>
template <typename key_t, typename value_t>
class LRU {
public:
typedef typename std::pair<key_t, value_t> key_value_pair_t;
......@@ -38,7 +37,8 @@ class LRU {
explicit LRU(size_t max_size) : max_size_(max_size) {
}
void put(const key_t &key, const value_t &value) {
void
put(const key_t& key, const value_t& value) {
auto it = cache_items_map_.find(key);
cache_items_list_.push_front(key_value_pair_t(key, value));
if (it != cache_items_map_.end()) {
......@@ -55,7 +55,8 @@ class LRU {
}
}
const value_t &get(const key_t &key) {
const value_t&
get(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it == cache_items_map_.end()) {
throw std::range_error("There is no such key in cache");
......@@ -65,7 +66,8 @@ class LRU {
}
}
void erase(const key_t &key) {
void
erase(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it != cache_items_map_.end()) {
cache_items_list_.erase(it->second);
......@@ -73,32 +75,39 @@ class LRU {
}
}
bool exists(const key_t &key) const {
bool
exists(const key_t& key) const {
return cache_items_map_.find(key) != cache_items_map_.end();
}
size_t size() const {
size_t
size() const {
return cache_items_map_.size();
}
list_iterator_t begin() {
list_iterator_t
begin() {
iter_ = cache_items_list_.begin();
return iter_;
}
list_iterator_t end() {
list_iterator_t
end() {
return cache_items_list_.end();
}
reverse_list_iterator_t rbegin() {
reverse_list_iterator_t
rbegin() {
return cache_items_list_.rbegin();
}
reverse_list_iterator_t rend() {
reverse_list_iterator_t
rend() {
return cache_items_list_.rend();
}
void clear() {
void
clear() {
cache_items_list_.clear();
cache_items_map_.clear();
}
......@@ -110,7 +119,6 @@ class LRU {
list_iterator_t iter_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
cpp/src/config/ConfigMgr.cpp
浏览文件 @ 35cfea92
......@@ -22,12 +22,12 @@ namespace zilliz {
namespace milvus {
namespace server {
ConfigMgr *
ConfigMgr*
ConfigMgr::GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/config/ConfigMgr.h
浏览文件 @ 35cfea92
......@@ -17,8 +17,8 @@
#pragma once
#include "utils/Error.h"
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
......@@ -42,16 +42,22 @@ namespace server {
class ConfigMgr {
public:
static ConfigMgr *GetInstance();
virtual ErrorCode LoadConfigFile(const std::string &filename) = 0;
virtual void Print() const = 0;//will be deleted
virtual std::string DumpString() const = 0;
virtual const ConfigNode &GetRootNode() const = 0;
virtual ConfigNode &GetRootNode() = 0;
static ConfigMgr*
GetInstance();
virtual ErrorCode
LoadConfigFile(const std::string& filename) = 0;
virtual void
Print() const = 0; // will be deleted
virtual std::string
DumpString() const = 0;
virtual const ConfigNode&
GetRootNode() const = 0;
virtual ConfigNode&
GetRootNode() = 0;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/config/ConfigNode.cpp
浏览文件 @ 35cfea92
......@@ -19,51 +19,51 @@
#include "utils/Error.h"
#include "utils/Log.h"
#include <algorithm>
#include <sstream>
#include <string>
#include <algorithm>
namespace zilliz {
namespace milvus {
namespace server {
void
ConfigNode::Combine(const ConfigNode &target) {
const std::map<std::string, std::string> &kv = target.GetConfig();
ConfigNode::Combine(const ConfigNode& target) {
const std::map<std::string, std::string>& kv = target.GetConfig();
for (auto itr = kv.begin(); itr != kv.end(); ++itr) {
config_[itr->first] = itr->second;
}
const std::map<std::string, std::vector<std::string> > &sequences = target.GetSequences();
const std::map<std::string, std::vector<std::string> >& sequences = target.GetSequences();
for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) {
sequences_[itr->first] = itr->second;
}
const std::map<std::string, ConfigNode> &children = target.GetChildren();
const std::map<std::string, ConfigNode>& children = target.GetChildren();
for (auto itr = children.begin(); itr != children.end(); ++itr) {
children_[itr->first] = itr->second;
}
}
//key/value pair config
// key/value pair config
void
ConfigNode::SetValue(const std::string &key, const std::string &value) {
ConfigNode::SetValue(const std::string& key, const std::string& value) {
config_[key] = value;
}
std::string
ConfigNode::GetValue(const std::string &param_key, const std::string &default_val) const {
ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const {
auto ref = config_.find(param_key);
if (ref != config_.end()) {
return ref->second;
}
//THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
// THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
return default_val;
}
bool
ConfigNode::GetBoolValue(const std::string &param_key, bool default_val) const {
ConfigNode::GetBoolValue(const std::string& param_key, bool default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
std::transform(val.begin(), val.end(), val.begin(), ::tolower);
......@@ -74,17 +74,17 @@ ConfigNode::GetBoolValue(const std::string &param_key, bool default_val) const {
}
int32_t
ConfigNode::GetInt32Value(const std::string &param_key, int32_t default_val) const {
ConfigNode::GetInt32Value(const std::string& param_key, int32_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return (int32_t) std::strtol(val.c_str(), nullptr, 10);
return (int32_t)std::strtol(val.c_str(), nullptr, 10);
} else {
return default_val;
}
}
int64_t
ConfigNode::GetInt64Value(const std::string &param_key, int64_t default_val) const {
ConfigNode::GetInt64Value(const std::string& param_key, int64_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtol(val.c_str(), nullptr, 10);
......@@ -94,7 +94,7 @@ ConfigNode::GetInt64Value(const std::string &param_key, int64_t default_val) con
}
float
ConfigNode::GetFloatValue(const std::string &param_key, float default_val) const {
ConfigNode::GetFloatValue(const std::string& param_key, float default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtof(val.c_str(), nullptr);
......@@ -104,7 +104,7 @@ ConfigNode::GetFloatValue(const std::string &param_key, float default_val) const
}
double
ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) const {
ConfigNode::GetDoubleValue(const std::string& param_key, double default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtod(val.c_str(), nullptr);
......@@ -113,7 +113,7 @@ ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) con
}
}
const std::map<std::string, std::string> &
const std::map<std::string, std::string>&
ConfigNode::GetConfig() const {
return config_;
}
......@@ -123,14 +123,14 @@ ConfigNode::ClearConfig() {
config_.clear();
}
//key/object config
// key/object config
void
ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) {
ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) {
children_[type_name] = config;
}
ConfigNode
ConfigNode::GetChild(const std::string &type_name) const {
ConfigNode::GetChild(const std::string& type_name) const {
auto ref = children_.find(type_name);
if (ref != children_.end()) {
return ref->second;
......@@ -140,20 +140,20 @@ ConfigNode::GetChild(const std::string &type_name) const {
return nc;
}
ConfigNode &
ConfigNode::GetChild(const std::string &type_name) {
ConfigNode&
ConfigNode::GetChild(const std::string& type_name) {
return children_[type_name];
}
void
ConfigNode::GetChildren(ConfigNodeArr &arr) const {
ConfigNode::GetChildren(ConfigNodeArr& arr) const {
arr.clear();
for (auto ref : children_) {
arr.push_back(ref.second);
}
}
const std::map<std::string, ConfigNode> &
const std::map<std::string, ConfigNode>&
ConfigNode::GetChildren() const {
return children_;
}
......@@ -163,14 +163,14 @@ ConfigNode::ClearChildren() {
children_.clear();
}
//key/sequence config
// key/sequence config
void
ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) {
ConfigNode::AddSequenceItem(const std::string& key, const std::string& item) {
sequences_[key].push_back(item);
}
std::vector<std::string>
ConfigNode::GetSequence(const std::string &key) const {
ConfigNode::GetSequence(const std::string& key) const {
auto itr = sequences_.find(key);
if (itr != sequences_.end()) {
return itr->second;
......@@ -180,7 +180,7 @@ ConfigNode::GetSequence(const std::string &key) const {
}
}
const std::map<std::string, std::vector<std::string> > &
const std::map<std::string, std::vector<std::string> >&
ConfigNode::GetSequences() const {
return sequences_;
}
......@@ -191,40 +191,40 @@ ConfigNode::ClearSequences() {
}
void
ConfigNode::PrintAll(const std::string &prefix) const {
for (auto &elem : config_) {
ConfigNode::PrintAll(const std::string& prefix) const {
for (auto& elem : config_) {
SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second;
}
for (auto &elem : sequences_) {
for (auto& elem : sequences_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
for (auto &str : elem.second) {
for (auto& str : elem.second) {
SERVER_LOG_INFO << prefix << " - " << str;
}
}
for (auto &elem : children_) {
for (auto& elem : children_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
elem.second.PrintAll(prefix + " ");
}
}
std::string
ConfigNode::DumpString(const std::string &prefix) const {
ConfigNode::DumpString(const std::string& prefix) const {
std::stringstream str_buffer;
const std::string endl = "\n";
for (auto &elem : config_) {
for (auto& elem : config_) {
str_buffer << prefix << elem.first << ": " << elem.second << endl;
}
for (auto &elem : sequences_) {
for (auto& elem : sequences_) {
str_buffer << prefix << elem.first << ": " << endl;
for (auto &str : elem.second) {
for (auto& str : elem.second) {
str_buffer << prefix + " - " << str << endl;
}
}
for (auto &elem : children_) {
for (auto& elem : children_) {
str_buffer << prefix << elem.first << ": " << endl;
str_buffer << elem.second.DumpString(prefix + " ") << endl;
}
......@@ -232,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const {
return str_buffer.str();
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/config/ConfigNode.h
浏览文件 @ 35cfea92
......@@ -17,9 +17,9 @@
#pragma once
#include <vector>
#include <string>
#include <map>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -30,39 +30,61 @@ typedef std::vector<ConfigNode> ConfigNodeArr;
class ConfigNode {
public:
void Combine(const ConfigNode &target);
void
Combine(const ConfigNode& target);
//key/value pair config
void SetValue(const std::string &key, const std::string &value);
// key/value pair config
void
SetValue(const std::string& key, const std::string& value);
std::string GetValue(const std::string &param_key, const std::string &default_val = "") const;
bool GetBoolValue(const std::string &param_key, bool default_val = false) const;
int32_t GetInt32Value(const std::string &param_key, int32_t default_val = 0) const;
int64_t GetInt64Value(const std::string &param_key, int64_t default_val = 0) const;
float GetFloatValue(const std::string &param_key, float default_val = 0.0) const;
double GetDoubleValue(const std::string &param_key, double default_val = 0.0) const;
std::string
GetValue(const std::string& param_key, const std::string& default_val = "") const;
bool
GetBoolValue(const std::string& param_key, bool default_val = false) const;
int32_t
GetInt32Value(const std::string& param_key, int32_t default_val = 0) const;
int64_t
GetInt64Value(const std::string& param_key, int64_t default_val = 0) const;
float
GetFloatValue(const std::string& param_key, float default_val = 0.0) const;
double
GetDoubleValue(const std::string& param_key, double default_val = 0.0) const;
const std::map<std::string, std::string> &GetConfig() const;
void ClearConfig();
const std::map<std::string, std::string>&
GetConfig() const;
void
ClearConfig();
//key/object config
void AddChild(const std::string &type_name, const ConfigNode &config);
ConfigNode GetChild(const std::string &type_name) const;
ConfigNode &GetChild(const std::string &type_name);
void GetChildren(ConfigNodeArr &arr) const;
// key/object config
void
AddChild(const std::string& type_name, const ConfigNode& config);
ConfigNode
GetChild(const std::string& type_name) const;
ConfigNode&
GetChild(const std::string& type_name);
void
GetChildren(ConfigNodeArr& arr) const;
const std::map<std::string, ConfigNode> &GetChildren() const;
void ClearChildren();
const std::map<std::string, ConfigNode>&
GetChildren() const;
void
ClearChildren();
//key/sequence config
void AddSequenceItem(const std::string &key, const std::string &item);
std::vector<std::string> GetSequence(const std::string &key) const;
// key/sequence config
void
AddSequenceItem(const std::string& key, const std::string& item);
std::vector<std::string>
GetSequence(const std::string& key) const;
const std::map<std::string, std::vector<std::string> > &GetSequences() const;
void ClearSequences();
const std::map<std::string, std::vector<std::string> >&
GetSequences() const;
void
ClearSequences();
void PrintAll(const std::string &prefix = "") const;
std::string DumpString(const std::string &prefix = "") const;
void
PrintAll(const std::string& prefix = "") const;
std::string
DumpString(const std::string& prefix = "") const;
private:
std::map<std::string, std::string> config_;
......@@ -70,6 +92,6 @@ class ConfigNode {
std::map<std::string, std::vector<std::string> > sequences_;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/config/YamlConfigMgr.cpp
浏览文件 @ 35cfea92
......@@ -25,7 +25,7 @@ namespace milvus {
namespace server {
ErrorCode
YamlConfigMgr::LoadConfigFile(const std::string &filename) {
YamlConfigMgr::LoadConfigFile(const std::string& filename) {
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
......@@ -36,8 +36,7 @@ YamlConfigMgr::LoadConfigFile(const std::string &filename) {
try {
node_ = YAML::LoadFile(filename);
LoadConfigNode(node_, config_);
}
catch (YAML::Exception &e) {
} catch (YAML::Exception& e) {
SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what());
return SERVER_UNEXPECTED_ERROR;
}
......@@ -56,20 +55,18 @@ YamlConfigMgr::DumpString() const {
return config_.DumpString("");
}
const ConfigNode &
const ConfigNode&
YamlConfigMgr::GetRootNode() const {
return config_;
}
ConfigNode &
ConfigNode&
YamlConfigMgr::GetRootNode() {
return config_;
}
bool
YamlConfigMgr::SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config) {
YamlConfigMgr::SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config) {
if (node[key].IsDefined()) {
config.SetValue(key, node[key].as<std::string>());
return true;
......@@ -78,9 +75,7 @@ YamlConfigMgr::SetConfigValue(const YAML::Node &node,
}
bool
YamlConfigMgr::SetChildConfig(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
YamlConfigMgr::SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
if (node[child_name].IsDefined()) {
ConfigNode sub_config;
LoadConfigNode(node[child_name], sub_config);
......@@ -91,9 +86,7 @@ YamlConfigMgr::SetChildConfig(const YAML::Node &node,
}
bool
YamlConfigMgr::SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
YamlConfigMgr::SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
if (node[child_name].IsDefined()) {
size_t cnt = node[child_name].size();
for (size_t i = 0; i < cnt; i++) {
......@@ -105,7 +98,7 @@ YamlConfigMgr::SetSequence(const YAML::Node &node,
}
void
YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) {
YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
std::string key;
for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) {
if (!it->first.IsNull()) {
......@@ -121,6 +114,6 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) {
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/config/YamlConfigMgr.h
浏览文件 @ 35cfea92
......@@ -21,8 +21,8 @@
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
#include <yaml-cpp/yaml.h>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -30,34 +30,36 @@ namespace server {
class YamlConfigMgr : public ConfigMgr {
public:
virtual ErrorCode LoadConfigFile(const std::string &filename);
virtual void Print() const;
virtual std::string DumpString() const;
virtual ErrorCode
LoadConfigFile(const std::string& filename);
virtual void
Print() const;
virtual std::string
DumpString() const;
virtual const ConfigNode &GetRootNode() const;
virtual ConfigNode &GetRootNode();
virtual const ConfigNode&
GetRootNode() const;
virtual ConfigNode&
GetRootNode();
private:
bool SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config);
bool
SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config);
bool SetChildConfig(const YAML::Node &node,
const std::string &name,
ConfigNode &config);
bool
SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config);
bool
SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config);
SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config);
void LoadConfigNode(const YAML::Node &node, ConfigNode &config);
void
LoadConfigNode(const YAML::Node& node, ConfigNode& config);
private:
YAML::Node node_;
ConfigNode config_;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/core/CMakeLists.txt
浏览文件 @ 35cfea92
......@@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE)
endif(NOT CMAKE_BUILD_TYPE)
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -fopenmp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -fopenmp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
endif()
MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
......
cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp
浏览文件 @ 35cfea92
......@@ -24,6 +24,7 @@
#include "NSG.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Log.h"
#include "knowhere/common/Timer.h"
#include "NSGHelper.h"
......@@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con
for (int i = 0; i < ntotal; ++i) {
total_degree += nsg[i].size();
}
std::cout << "graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024;
std::cout << "average degree: " << total_degree / ntotal;
KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m";
KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal;
/////
is_trained = true;
......
cpp/src/core/test/CMakeLists.txt
浏览文件 @ 35cfea92
......@@ -29,7 +29,7 @@ set(util_srcs
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp
utils.cpp
${CORE_SOURCE_DIR}/test/utils.cpp
)
#<IVF-TEST>
......@@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<IDMAP-TEST>
set(idmap_srcs
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp
)
if(NOT TARGET test_idmap)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs})
add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs})
endif()
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
......@@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest)
install(TARGETS test_kdt DESTINATION unittest)
#add_subdirectory(faiss_ori)
#add_subdirectory(test_nsg)
add_subdirectory(test_nsg)
cpp/src/core/test/test_nsg/CMakeLists.txt
浏览文件 @ 35cfea92
##############################
include_directories(/usr/local/include/gperftools)
link_directories(/usr/local/lib)
#include_directories(/usr/local/include/gperftools)
#link_directories(/usr/local/lib)
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
......@@ -13,29 +13,15 @@ else ()
endif ()
message(${OpenMP_CXX_FLAGS})
include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
include_directories(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg)
aux_source_directory(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src)
set(interface_src
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
../utils.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp
)
if(NOT TARGET test_nsg)
add_executable(test_nsg
test_nsg.cpp
${interface_src}
${nsg_src}
${util_srcs}
)
add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs})
endif()
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
......
cpp/src/core/test/test_nsg/test_nsg.cpp
浏览文件 @ 35cfea92
......@@ -19,10 +19,11 @@
#include <gtest/gtest.h>
#include <memory>
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/nsg_index.h"
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/FaissBaseIndex.h"
#include "knowhere/index/vector_index/IndexNSG.h"
#include "knowhere/index/vector_index/nsg/NSGIO.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "../utils.h"
......@@ -32,16 +33,31 @@ using ::testing::TestWithParam;
using ::testing::Values;
using ::testing::Combine;
constexpr int64_t DEVICE_ID = 0;
constexpr int64_t DEVICE_ID = 1;
class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Config, Config>> {
class NSGInterfaceTest : public DataGen, public ::testing::Test {
protected:
void SetUp() override {
//Init_with_default();
FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2);
Generate(256, 10000, 1);
Generate(256, 1000000, 1);
index_ = std::make_shared<NSG>();
std::tie(train_cfg, search_cfg) = GetParam();
auto tmp_conf = std::make_shared<NSGCfg>();
tmp_conf->gpu_id = DEVICE_ID;
tmp_conf->knng = 100;
tmp_conf->nprobe = 32;
tmp_conf->nlist = 16384;
tmp_conf->search_length = 60;
tmp_conf->out_degree = 70;
tmp_conf->candidate_pool_size = 500;
tmp_conf->metric_type = METRICTYPE::L2;
train_conf = tmp_conf;
auto tmp2_conf = std::make_shared<NSGCfg>();
tmp2_conf->k = k;
tmp2_conf->search_length = 30;
search_conf = tmp2_conf;
}
void TearDown() override {
......@@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Confi
protected:
std::shared_ptr<NSG> index_;
Config train_cfg;
Config search_cfg;
Config train_conf;
Config search_conf;
};
INSTANTIATE_TEST_CASE_P(NSGparameters, NSGInterfaceTest,
Values(std::make_tuple(
// search length > out_degree
Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"},
{"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}},
Config::object{{"k", 20}, {"search_length", 30}}))
);
void AssertAnns(const DatasetPtr &result,
const int &nq,
const int &k) {
......@@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result,
}
}
TEST_P(NSGInterfaceTest, basic_test) {
TEST_F(NSGInterfaceTest, basic_test) {
assert(!xb.empty());
auto model = index_->Train(base_dataset, train_cfg);
auto result = index_->Search(query_dataset, search_cfg);
auto model = index_->Train(base_dataset, train_conf);
auto result = index_->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
auto binaryset = index_->Serialize();
auto new_index = std::make_shared<NSG>();
new_index->Load(binaryset);
auto new_result = new_index->Search(query_dataset, Config::object{{"k", k}});
auto new_result = new_index->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
ASSERT_EQ(index_->Count(), nb);
......
cpp/src/db/Constants.h
浏览文件 @ 35cfea92
......@@ -36,6 +36,6 @@ static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/DB.h
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "Options.h"
......@@ -23,8 +22,8 @@
#include "meta/Meta.h"
#include "utils/Status.h"
#include <string>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
......@@ -36,47 +35,65 @@ class Env;
class DB {
public:
DB() = default;
DB(const DB &) = delete;
DB &operator=(const DB &) = delete;
DB(const DB&) = delete;
DB&
operator=(const DB&) = delete;
virtual ~DB() = default;
virtual Status Start() = 0;
virtual Status Stop() = 0;
virtual Status CreateTable(meta::TableSchema &table_schema_) = 0;
virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0;
virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema> &table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0;
virtual Status PreloadTable(const std::string &table_id) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropIndex(const std::string &table_id) = 0;
virtual Status DropAll() = 0;
}; // DB
virtual Status
Start() = 0;
virtual Status
Stop() = 0;
virtual Status
CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
virtual Status
DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status
PreloadTable(const std::string& table_id) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status
InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status
CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status
DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status
DropIndex(const std::string& table_id) = 0;
virtual Status
DropAll() = 0;
}; // DB
using DBPtr = std::shared_ptr<DB>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/DBFactory.cpp
浏览文件 @ 35cfea92
......@@ -15,18 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "db/DBFactory.h"
#include "DBImpl.h"
#include "utils/Exception.h"
#include "meta/MetaFactory.h"
#include "meta/SqliteMetaImpl.h"
#include "meta/MySQLMetaImpl.h"
#include "meta/SqliteMetaImpl.h"
#include "utils/Exception.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <sstream>
#include <string>
namespace zilliz {
......@@ -42,10 +41,10 @@ DBFactory::BuildOption() {
}
DBPtr
DBFactory::Build(const DBOptions &options) {
DBFactory::Build(const DBOptions& options) {
return std::make_shared<DBImpl>(options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/DBFactory.h
浏览文件 @ 35cfea92
......@@ -20,8 +20,8 @@
#include "DB.h"
#include "Options.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -29,11 +29,13 @@ namespace engine {
class DBFactory {
public:
static DBOptions BuildOption();
static DBOptions
BuildOption();
static DBPtr Build(const DBOptions &options);
static DBPtr
Build(const DBOptions& options);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/DBImpl.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/db/DBImpl.h
浏览文件 @ 35cfea92
......@@ -19,18 +19,18 @@
#include "DB.h"
#include "Types.h"
#include "utils/ThreadPool.h"
#include "src/db/insert/MemManager.h"
#include "utils/ThreadPool.h"
#include <mutex>
#include <condition_variable>
#include <memory>
#include <atomic>
#include <thread>
#include <condition_variable>
#include <list>
#include <memory>
#include <mutex>
#include <set>
#include <vector>
#include <string>
#include <thread>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -44,92 +44,101 @@ class Meta;
class DBImpl : public DB {
public:
explicit DBImpl(const DBOptions &options);
explicit DBImpl(const DBOptions& options);
~DBImpl();
Status Start() override;
Status Stop() override;
Status DropAll() override;
Status
Start() override;
Status
Stop() override;
Status
DropAll() override;
Status CreateTable(meta::TableSchema &table_schema) override;
Status
CreateTable(meta::TableSchema& table_schema) override;
Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override;
Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) override;
Status DescribeTable(meta::TableSchema &table_schema) override;
Status
DescribeTable(meta::TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<meta::TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) override;
Status PreloadTable(const std::string &table_id) override;
Status
PreloadTable(const std::string& table_id) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag);
Status
UpdateTableFlag(const std::string& table_id, int64_t flag);
Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override;
Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status
InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override;
Status CreateIndex(const std::string &table_id, const TableIndex &index) override;
Status
CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status DescribeIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DropIndex(const std::string &table_id) override;
Status
DropIndex(const std::string& table_id) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
QueryResults &results) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) override;
Status Query(const std::string &table_id,
const std::vector<std::string> &file_ids,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
private:
Status QueryAsync(const std::string &table_id,
const meta::TableFilesSchema &files,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results);
void BackgroundTimerTask();
void WaitMergeFileFinish();
void WaitBuildIndexFinish();
void StartMetricTask();
void StartCompactionTask();
Status MergeFiles(const std::string &table_id,
const meta::DateT &date,
const meta::TableFilesSchema &files);
Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask(bool force = false);
void BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema &);
Status MemSerialize();
Status
QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results);
void
BackgroundTimerTask();
void
WaitMergeFileFinish();
void
WaitBuildIndexFinish();
void
StartMetricTask();
void
StartCompactionTask();
Status
MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files);
Status
BackgroundMergeFiles(const std::string& table_id);
void
BackgroundCompaction(std::set<std::string> table_ids);
void
StartBuildIndexTask(bool force = false);
void
BackgroundBuildIndex();
Status
BuildIndex(const meta::TableFileSchema&);
Status
MemSerialize();
private:
const DBOptions options_;
......@@ -152,9 +161,8 @@ class DBImpl : public DB {
std::list<std::future<void>> index_thread_results_;
std::mutex build_index_mutex_;
}; // DBImpl
}; // DBImpl
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/IDGenerator.cpp
浏览文件 @ 35cfea92
......@@ -17,8 +17,8 @@
#include "db/IDGenerator.h"
#include <chrono>
#include <assert.h>
#include <chrono>
#include <iostream>
namespace zilliz {
......@@ -32,13 +32,12 @@ constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
IDNumber
SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO;
}
void
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
if (n > MAX_IDS_PER_MICRO) {
NextIDNumbers(n - MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids);
......@@ -49,8 +48,7 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
}
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
micros *= MAX_IDS_PER_MICRO;
for (int pos = 0; pos < n; ++pos) {
......@@ -59,11 +57,11 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
}
void
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) {
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
ids.clear();
NextIDNumbers(n, ids);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/IDGenerator.h
浏览文件 @ 35cfea92
......@@ -28,16 +28,14 @@ namespace engine {
class IDGenerator {
public:
virtual
IDNumber GetNextIDNumber() = 0;
virtual IDNumber
GetNextIDNumber() = 0;
virtual void
GetNextIDNumbers(size_t n, IDNumbers &ids) = 0;
virtual
~IDGenerator() = 0;
}; // IDGenerator
GetNextIDNumbers(size_t n, IDNumbers& ids) = 0;
virtual ~IDGenerator() = 0;
}; // IDGenerator
class SimpleIDGenerator : public IDGenerator {
public:
......@@ -47,16 +45,15 @@ class SimpleIDGenerator : public IDGenerator {
GetNextIDNumber() override;
void
GetNextIDNumbers(size_t n, IDNumbers &ids) override;
GetNextIDNumbers(size_t n, IDNumbers& ids) override;
private:
void
NextIDNumbers(size_t n, IDNumbers &ids);
NextIDNumbers(size_t n, IDNumbers& ids);
static constexpr size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator
}; // SimpleIDGenerator
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/Options.cpp
浏览文件 @ 35cfea92
......@@ -19,28 +19,28 @@
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <assert.h>
#include <stdlib.h>
#include <boost/algorithm/string.hpp>
namespace zilliz {
namespace milvus {
namespace engine {
ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) {
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
ParseType(type);
ParseCritirias(criterias);
}
void
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) {
for (auto &pair : criterial) {
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
for (auto& pair : criterial) {
criterias_[pair.first] = pair.second;
}
}
void
ArchiveConf::ParseCritirias(const std::string &criterias) {
ArchiveConf::ParseCritirias(const std::string& criterias) {
std::stringstream ss(criterias);
std::vector<std::string> tokens;
......@@ -50,7 +50,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
return;
}
for (auto &token : tokens) {
for (auto& token : tokens) {
if (token.empty()) {
continue;
}
......@@ -68,13 +68,11 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
try {
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
}
catch (std::out_of_range &) {
} catch (std::out_of_range&) {
std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
catch (...) {
} catch (...) {
std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
......@@ -83,7 +81,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
}
void
ArchiveConf::ParseType(const std::string &type) {
ArchiveConf::ParseType(const std::string& type) {
if (type != "delete" && type != "swap") {
std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg);
......@@ -91,6 +89,6 @@ ArchiveConf::ParseType(const std::string &type) {
type_ = type;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/Options.h
浏览文件 @ 35cfea92
......@@ -19,9 +19,9 @@
#include "Constants.h"
#include <string>
#include <memory>
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
......@@ -30,27 +30,32 @@ namespace engine {
class Env;
static const char *ARCHIVE_CONF_DISK = "disk";
static const char *ARCHIVE_CONF_DAYS = "days";
static const char* ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;
explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string());
explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string());
const std::string &GetType() const {
const std::string&
GetType() const {
return type_;
}
const CriteriaT GetCriterias() const {
const CriteriaT
GetCriterias() const {
return criterias_;
}
void SetCriterias(const ArchiveConf::CriteriaT &criterial);
void
SetCriterias(const ArchiveConf::CriteriaT& criterial);
private:
void ParseCritirias(const std::string &type);
void ParseType(const std::string &criterias);
void
ParseCritirias(const std::string& type);
void
ParseType(const std::string& criterias);
std::string type_;
CriteriaT criterias_;
......@@ -61,14 +66,10 @@ struct DBMetaOptions {
std::vector<std::string> slave_paths_;
std::string backend_uri_;
ArchiveConf archive_conf_ = ArchiveConf("delete");
}; // DBMetaOptions
}; // DBMetaOptions
struct DBOptions {
typedef enum {
SINGLE = 0,
CLUSTER_READONLY,
CLUSTER_WRITABLE
} MODE;
typedef enum { SINGLE = 0, CLUSTER_READONLY, CLUSTER_WRITABLE } MODE;
uint16_t merge_trigger_number_ = 2;
DBMetaOptions meta_;
......@@ -76,9 +77,8 @@ struct DBOptions {
size_t insert_buffer_size_ = 4 * ONE_GB;
bool insert_cache_immediately_ = false;
}; // Options
}; // Options
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/Types.h
浏览文件 @ 35cfea92
......@@ -19,27 +19,27 @@
#include "db/engine/ExecutionEngine.h"
#include <vector>
#include <stdint.h>
#include <utility>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
typedef int64_t IDNumber;
typedef IDNumber *IDNumberPtr;
typedef IDNumber* IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int) EngineType::FAISS_IDMAP;
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t metric_type_ = (int) MetricType::L2;
int32_t metric_type_ = (int)MetricType::L2;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/Utils.cpp
浏览文件 @ 35cfea92
......@@ -19,11 +19,11 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <mutex>
#include <boost/filesystem.hpp>
#include <chrono>
#include <mutex>
#include <regex>
#include <vector>
#include <boost/filesystem.hpp>
namespace zilliz {
namespace milvus {
......@@ -32,20 +32,20 @@ namespace utils {
namespace {
const char *TABLES_FOLDER = "/tables/";
const char* TABLES_FOLDER = "/tables/";
uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex;
std::string
ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) {
ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
std::string partition_path = table_path + "/" + std::to_string(table_file.date_);
return partition_path;
}
std::string
GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) {
GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_;
uint64_t index = 0;
......@@ -70,19 +70,18 @@ GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSche
return ConstructParentFolder(target_path, table_file);
}
} // namespace
} // namespace
int64_t
GetMicroSecTimeStamp() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
return micros;
}
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
......@@ -91,7 +90,7 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
return status;
}
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) {
......@@ -104,17 +103,16 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
}
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) {
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_);
for (auto &path : paths) {
for (auto& path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
if (force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if (boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
} else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
}
......@@ -124,7 +122,7 @@ DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool
}
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
......@@ -139,14 +137,14 @@ CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_f
}
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
} else {
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file);
file_path = parent_path + "/" + table_file.file_id_;
if (boost::filesystem::exists(file_path)) {
......@@ -157,28 +155,26 @@ GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file
}
std::string msg = "Table file doesn't exist: " + file_path;
ENGINE_LOG_ERROR << msg << " in path: " << options.path_
<< " for table: " << table_file.table_id_;
ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_;
return Status(DB_ERROR, msg);
}
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
return Status::OK();
}
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_
&& index1.metric_type_ == index2.metric_type_;
IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ &&
index1.metric_type_ == index2.metric_type_;
}
meta::DateT
GetDate(const std::time_t &t, int day_delta) {
GetDate(const std::time_t& t, int day_delta) {
struct tm ltm;
localtime_r(&t, &ltm);
if (day_delta > 0) {
......@@ -211,20 +207,15 @@ GetDate() {
// URI format: dialect://username:password@host:port/database
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std::string dialect_regex = "(.*)";
std::string username_tegex = "(.*)";
std::string password_regex = "(.*)";
std::string host_regex = "(.*)";
std::string port_regex = "(.*)";
std::string db_name_regex = "(.*)";
std::string uri_regex_str =
dialect_regex + "\\:\\/\\/" +
username_tegex + "\\:" +
password_regex + "\\@" +
host_regex + "\\:" +
port_regex + "\\/" +
db_name_regex;
std::string uri_regex_str = dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" +
host_regex + "\\:" + port_regex + "\\/" + db_name_regex;
std::regex uri_regex(uri_regex_str);
std::smatch pieces_match;
......@@ -237,7 +228,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
info.port_ = pieces_match[5].str();
info.db_name_ = pieces_match[6].str();
//TODO: verify host, port...
// TODO: verify host, port...
} else {
return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri);
}
......@@ -245,7 +236,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
return Status::OK();
}
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/Utils.h
浏览文件 @ 35cfea92
......@@ -18,11 +18,11 @@
#pragma once
#include "Options.h"
#include "db/meta/MetaTypes.h"
#include "db/Types.h"
#include "db/meta/MetaTypes.h"
#include <string>
#include <ctime>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -33,22 +33,22 @@ int64_t
GetMicroSecTimeStamp();
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id);
CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true);
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2);
IsSameIndex(const TableIndex& index1, const TableIndex& index2);
meta::DateT
GetDate(const std::time_t &t, int day_delta = 0);
GetDate(const std::time_t& t, int day_delta = 0);
meta::DateT
GetDate();
meta::DateT
......@@ -64,9 +64,9 @@ struct MetaUriInfo {
};
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info);
ParseMetaUri(const std::string& uri, MetaUriInfo& info);
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/engine/EngineFactory.cpp
浏览文件 @ 35cfea92
......@@ -26,17 +26,14 @@ namespace milvus {
namespace engine {
ExecutionEnginePtr
EngineFactory::Build(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist) {
if (index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type;
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type;
ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
......@@ -44,6 +41,6 @@ EngineFactory::Build(uint16_t dimension,
return execution_engine_ptr;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/engine/EngineFactory.h
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "ExecutionEngine.h"
......@@ -29,14 +28,11 @@ namespace engine {
class EngineFactory {
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist);
static ExecutionEnginePtr
Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/engine/ExecutionEngine.h
浏览文件 @ 35cfea92
......@@ -19,9 +19,9 @@
#include "utils/Status.h"
#include <vector>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -43,52 +43,66 @@ enum class MetricType {
class ExecutionEngine {
public:
virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0;
virtual Status
AddWithIds(int64_t n, const float* xdata, const int64_t* xids) = 0;
virtual size_t Count() const = 0;
virtual size_t
Count() const = 0;
virtual size_t Size() const = 0;
virtual size_t
Size() const = 0;
virtual size_t Dimension() const = 0;
virtual size_t
Dimension() const = 0;
virtual size_t PhysicalSize() const = 0;
virtual size_t
PhysicalSize() const = 0;
virtual Status Serialize() = 0;
virtual Status
Serialize() = 0;
virtual Status Load(bool to_cache = true) = 0;
virtual Status
Load(bool to_cache = true) = 0;
virtual Status CopyToGpu(uint64_t device_id) = 0;
virtual Status
CopyToGpu(uint64_t device_id) = 0;
virtual Status CopyToCpu() = 0;
virtual Status
CopyToCpu() = 0;
virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
virtual std::shared_ptr<ExecutionEngine>
Clone() = 0;
virtual Status Merge(const std::string &location) = 0;
virtual Status
Merge(const std::string& location) = 0;
virtual Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const = 0;
virtual Status
Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string &location, EngineType engine_type) = 0;
virtual std::shared_ptr<ExecutionEngine>
BuildIndex(const std::string& location, EngineType engine_type) = 0;
virtual Status Cache() = 0;
virtual Status
Cache() = 0;
virtual Status GpuCache(uint64_t gpu_id) = 0;
virtual Status
GpuCache(uint64_t gpu_id) = 0;
virtual Status Init() = 0;
virtual Status
Init() = 0;
virtual EngineType IndexEngineType() const = 0;
virtual EngineType
IndexEngineType() const = 0;
virtual MetricType IndexMetricType() const = 0;
virtual MetricType
IndexMetricType() const = 0;
virtual std::string GetLocation() const = 0;
virtual std::string
GetLocation() const = 0;
};
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @ 35cfea92
......@@ -16,20 +16,20 @@
// under the License.
#include "db/engine/ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "src/wrapper/VecIndex.h"
#include "src/wrapper/VecImpl.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Config.h"
#include "wrapper/ConfAdapterMgr.h"
#include "wrapper/ConfAdapter.h"
#include "knowhere/common/Exception.h"
#include "server/Config.h"
#include "src/wrapper/VecImpl.h"
#include "src/wrapper/VecIndex.h"
#include "wrapper/ConfAdapter.h"
#include "wrapper/ConfAdapterMgr.h"
#include <stdexcept>
#include <utility>
......@@ -38,17 +38,9 @@ namespace zilliz {
namespace milvus {
namespace engine {
ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: location_(location),
dim_(dimension),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type,
MetricType metric_type, int32_t nlist)
: location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
if (!index_) {
throw Exception(DB_ERROR, "Could not create VecIndex");
......@@ -57,8 +49,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
TempMetaConf temp_conf;
temp_conf.gpu_id = gpu_num_;
temp_conf.dim = dimension;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());
auto conf = adapter->Match(temp_conf);
......@@ -68,16 +59,9 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
}
}
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: index_(std::move(index)),
location_(location),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type,
MetricType metric_type, int32_t nlist)
: index_(std::move(index)), location_(location), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
}
VecIndexPtr
......@@ -109,7 +93,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
}
Status
ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) {
ExecutionEngineImpl::AddWithIds(int64_t n, const float* xdata, const int64_t* xids) {
auto status = index_->Add(n, xdata, xids);
return status;
}
......@@ -125,7 +109,7 @@ ExecutionEngineImpl::Count() const {
size_t
ExecutionEngineImpl::Size() const {
return (size_t) (Count() * Dimension()) * sizeof(float);
return (size_t)(Count() * Dimension()) * sizeof(float);
}
size_t
......@@ -164,7 +148,7 @@ ExecutionEngineImpl::Load(bool to_cache) {
} else {
ENGINE_LOG_DEBUG << "Disk io from: " << location_;
}
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
......@@ -191,7 +175,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
try {
index_ = index_->CopyToGpu(device_id);
ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
......@@ -219,7 +203,7 @@ ExecutionEngineImpl::CopyToCpu() {
try {
index_ = index_->CopyToCpu();
ENGINE_LOG_DEBUG << "GPU to CPU";
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
......@@ -245,7 +229,7 @@ ExecutionEngineImpl::Clone() {
}
Status
ExecutionEngineImpl::Merge(const std::string &location) {
ExecutionEngineImpl::Merge(const std::string& location) {
if (location == location_) {
return Status(DB_ERROR, "Cannot Merge Self");
}
......@@ -257,7 +241,7 @@ ExecutionEngineImpl::Merge(const std::string &location) {
double physical_size = server::CommonUtil::GetFileSize(location);
server::CollectExecutionEngineMetrics metrics(physical_size);
to_merge = read_index(location);
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
......@@ -280,7 +264,7 @@ ExecutionEngineImpl::Merge(const std::string &location) {
}
ExecutionEnginePtr
ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) {
ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_type) {
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
......@@ -298,29 +282,23 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
temp_conf.gpu_id = gpu_num_;
temp_conf.dim = Dimension();
temp_conf.nlist = nlist_;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.size = Count();
auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType());
auto conf = adapter->Match(temp_conf);
auto status = to_index->BuildAll(Count(),
from_index->GetRawVectors(),
from_index->GetRawIds(),
conf);
if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
auto status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf);
if (!status.ok()) {
throw Exception(DB_ERROR, status.message());
}
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
}
Status
ExecutionEngineImpl::Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const {
ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances,
int64_t* labels) const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status(DB_ERROR, "index is null");
......@@ -362,13 +340,13 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
// TODO(linxj): remove.
Status
ExecutionEngineImpl::Init() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s;
return Status::OK();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/engine/ExecutionEngineImpl.h
浏览文件 @ 35cfea92
......@@ -29,71 +29,81 @@ namespace engine {
class ExecutionEngineImpl : public ExecutionEngine {
public:
ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType index_type,
MetricType metric_type,
ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override;
Status
AddWithIds(int64_t n, const float* xdata, const int64_t* xids) override;
size_t Count() const override;
size_t
Count() const override;
size_t Size() const override;
size_t
Size() const override;
size_t Dimension() const override;
size_t
Dimension() const override;
size_t PhysicalSize() const override;
size_t
PhysicalSize() const override;
Status Serialize() override;
Status
Serialize() override;
Status Load(bool to_cache) override;
Status
Load(bool to_cache) override;
Status CopyToGpu(uint64_t device_id) override;
Status
CopyToGpu(uint64_t device_id) override;
Status CopyToCpu() override;
Status
CopyToCpu() override;
ExecutionEnginePtr Clone() override;
ExecutionEnginePtr
Clone() override;
Status Merge(const std::string &location) override;
Status
Merge(const std::string& location) override;
Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const override;
Status
Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
ExecutionEnginePtr
BuildIndex(const std::string& location, EngineType engine_type) override;
Status Cache() override;
Status
Cache() override;
Status GpuCache(uint64_t gpu_id) override;
Status
GpuCache(uint64_t gpu_id) override;
Status Init() override;
Status
Init() override;
EngineType IndexEngineType() const override {
EngineType
IndexEngineType() const override {
return index_type_;
}
MetricType IndexMetricType() const override {
MetricType
IndexMetricType() const override {
return metric_type_;
}
std::string GetLocation() const override {
std::string
GetLocation() const override {
return location_;
}
private:
VecIndexPtr CreatetVecIndex(EngineType type);
VecIndexPtr
CreatetVecIndex(EngineType type);
VecIndexPtr Load(const std::string &location);
VecIndexPtr
Load(const std::string& location);
protected:
VecIndexPtr index_ = nullptr;
......@@ -107,6 +117,6 @@ class ExecutionEngineImpl : public ExecutionEngine {
int32_t gpu_num_ = 0;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemManager.h
浏览文件 @ 35cfea92
......@@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "utils/Status.h"
#include "db/Types.h"
#include "utils/Status.h"
#include <set>
#include <memory>
#include <set>
#include <string>
namespace zilliz {
......@@ -31,22 +30,27 @@ namespace engine {
class MemManager {
public:
virtual Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) = 0;
virtual Status
InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) = 0;
virtual Status Serialize(std::set<std::string> &table_ids) = 0;
virtual Status
Serialize(std::set<std::string>& table_ids) = 0;
virtual Status EraseMemVector(const std::string &table_id) = 0;
virtual Status
EraseMemVector(const std::string& table_id) = 0;
virtual size_t GetCurrentMutableMem() = 0;
virtual size_t
GetCurrentMutableMem() = 0;
virtual size_t GetCurrentImmutableMem() = 0;
virtual size_t
GetCurrentImmutableMem() = 0;
virtual size_t GetCurrentMem() = 0;
}; // MemManagerAbstract
virtual size_t
GetCurrentMem() = 0;
}; // MemManagerAbstract
using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemManagerImpl.cpp
浏览文件 @ 35cfea92
......@@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemManagerImpl.h"
#include "VectorSource.h"
#include "utils/Log.h"
#include "db/Constants.h"
#include "utils/Log.h"
#include <thread>
......@@ -28,7 +27,7 @@ namespace milvus {
namespace engine {
MemTablePtr
MemManagerImpl::GetMemByTable(const std::string &table_id) {
MemManagerImpl::GetMemByTable(const std::string& table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
......@@ -39,10 +38,7 @@ MemManagerImpl::GetMemByTable(const std::string &table_id) {
}
Status
MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
MemManagerImpl::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
while (GetCurrentMem() > options_.insert_buffer_size_) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
......@@ -53,10 +49,8 @@ MemManagerImpl::InsertVectors(const std::string &table_id_,
}
Status
MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors,
IDNumbers& vector_ids) {
MemTablePtr mem = GetMemByTable(table_id);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
......@@ -73,9 +67,9 @@ Status
MemManagerImpl::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map;
for (auto &kv : mem_id_map_) {
for (auto& kv : mem_id_map_) {
if (kv.second->Empty()) {
//empty table, no need to serialize
// empty table, no need to serialize
temp_map.insert(kv);
} else {
immu_mem_list_.push_back(kv.second);
......@@ -87,11 +81,11 @@ MemManagerImpl::ToImmutable() {
}
Status
MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
MemManagerImpl::Serialize(std::set<std::string>& table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
table_ids.clear();
for (auto &mem : immu_mem_list_) {
for (auto& mem : immu_mem_list_) {
mem->Serialize();
table_ids.insert(mem->GetTableId());
}
......@@ -100,16 +94,16 @@ MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
}
Status
MemManagerImpl::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache
MemManagerImpl::EraseMemVector(const std::string& table_id) {
{ // erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
}
{//erase MemVector from serialize cache
{ // erase MemVector from serialize cache
std::unique_lock<std::mutex> lock(serialization_mtx_);
MemList temp_list;
for (auto &mem : immu_mem_list_) {
for (auto& mem : immu_mem_list_) {
if (mem->GetTableId() != table_id) {
temp_list.push_back(mem);
}
......@@ -123,7 +117,7 @@ MemManagerImpl::EraseMemVector(const std::string &table_id) {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
for (auto &kv : mem_id_map_) {
for (auto& kv : mem_id_map_) {
auto memTable = kv.second;
total_mem += memTable->GetCurrentMem();
}
......@@ -133,7 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
for (auto &mem_table : immu_mem_list_) {
for (auto& mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
}
return total_mem;
......@@ -144,6 +138,6 @@ MemManagerImpl::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemManagerImpl.h
浏览文件 @ 35cfea92
......@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MemTable.h"
#include "MemManager.h"
#include "MemTable.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <map>
#include <set>
#include <vector>
#include <string>
#include <ctime>
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -39,29 +38,35 @@ class MemManagerImpl : public MemManager {
public:
using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {
MemManagerImpl(const meta::MetaPtr& meta, const DBOptions& options) : meta_(meta), options_(options) {
}
Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override;
Status
InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) override;
Status Serialize(std::set<std::string> &table_ids) override;
Status
Serialize(std::set<std::string>& table_ids) override;
Status EraseMemVector(const std::string &table_id) override;
Status
EraseMemVector(const std::string& table_id) override;
size_t GetCurrentMutableMem() override;
size_t
GetCurrentMutableMem() override;
size_t GetCurrentImmutableMem() override;
size_t
GetCurrentImmutableMem() override;
size_t GetCurrentMem() override;
size_t
GetCurrentMem() override;
private:
MemTablePtr GetMemByTable(const std::string &table_id);
MemTablePtr
GetMemByTable(const std::string& table_id);
Status InsertVectorsNoLock(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids);
Status ToImmutable();
Status
InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids);
Status
ToImmutable();
using MemIdMap = std::map<std::string, MemTablePtr>;
using MemList = std::vector<MemTablePtr>;
......@@ -71,8 +76,8 @@ class MemManagerImpl : public MemManager {
DBOptions options_;
std::mutex mutex_;
std::mutex serialization_mtx_;
}; // NewMemManager
}; // NewMemManager
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemMenagerFactory.cpp
浏览文件 @ 35cfea92
......@@ -17,26 +17,26 @@
#include "db/insert/MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <regex>
#include <memory>
#include <regex>
#include <sstream>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr
MemManagerFactory::Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options) {
MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
return std::make_shared<MemManagerImpl>(meta, options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemMenagerFactory.h
浏览文件 @ 35cfea92
......@@ -28,9 +28,10 @@ namespace engine {
class MemManagerFactory {
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
static MemManagerPtr
Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemTable.cpp
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemTable.h"
#include "utils/Log.h"
......@@ -26,16 +25,12 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemTable::MemTable(const std::string &table_id,
const meta::MetaPtr &meta,
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
MemTable::MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
}
Status
MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
MemTable::Add(VectorSourcePtr& source, IDNumbers& vector_ids) {
while (!source->AllAdded()) {
MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) {
......@@ -63,7 +58,7 @@ MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
}
void
MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
MemTable::GetCurrentMemTableFile(MemTableFilePtr& mem_table_file) {
mem_table_file = mem_table_file_list_.back();
}
......@@ -92,7 +87,7 @@ MemTable::Empty() {
return mem_table_file_list_.empty();
}
const std::string &
const std::string&
MemTable::GetTableId() const {
return table_id_;
}
......@@ -101,12 +96,12 @@ size_t
MemTable::GetCurrentMem() {
std::lock_guard<std::mutex> lock(mutex_);
size_t total_mem = 0;
for (auto &mem_table_file : mem_table_file_list_) {
for (auto& mem_table_file : mem_table_file_list_) {
total_mem += mem_table_file->GetCurrentMem();
}
return total_mem;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemTable.h
浏览文件 @ 35cfea92
......@@ -15,17 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MemTableFile.h"
#include "VectorSource.h"
#include "utils/Status.h"
#include <mutex>
#include <vector>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -35,21 +34,28 @@ class MemTable {
public:
using MemTableFileList = std::vector<MemTableFilePtr>;
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
Status Add(VectorSourcePtr &source, IDNumbers &vector_ids);
Status
Add(VectorSourcePtr& source, IDNumbers& vector_ids);
void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file);
void
GetCurrentMemTableFile(MemTableFilePtr& mem_table_file);
size_t GetTableFileCount();
size_t
GetTableFileCount();
Status Serialize();
Status
Serialize();
bool Empty();
bool
Empty();
const std::string &GetTableId() const;
const std::string&
GetTableId() const;
size_t GetCurrentMem();
size_t
GetCurrentMem();
private:
const std::string table_id_;
......@@ -61,10 +67,10 @@ class MemTable {
DBOptions options_;
std::mutex mutex_;
}; //MemTable
}; // MemTable
using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemTableFile.cpp
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemTableFile.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
......@@ -29,20 +28,14 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemTableFile::MemTableFile(const std::string &table_id,
const meta::MetaPtr &meta,
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
current_mem_ = 0;
auto status = CreateTableFile();
if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_,
(MetricType) table_file_schema_.metric_type_,
table_file_schema_.nlist_);
execution_engine_ = EngineFactory::Build(
table_file_schema_.dimension_, table_file_schema_.location_, (EngineType)table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_, table_file_schema_.nlist_);
}
}
......@@ -61,10 +54,11 @@ MemTableFile::CreateTableFile() {
}
Status
MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) {
MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
std::to_string(table_file_schema_.dimension_) + ", table_id = " +
table_file_schema_.table_id_;
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, "Not able to create table file");
}
......@@ -109,11 +103,11 @@ MemTableFile::Serialize() {
table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
table_file_schema_.row_count_ = execution_engine_->Count();
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
// if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
// else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX
: meta::TableFileSchema::RAW;
} else {
table_file_schema_.file_type_ = meta::TableFileSchema::RAW;
}
......@@ -130,6 +124,6 @@ MemTableFile::Serialize() {
return status;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/MemTableFile.h
浏览文件 @ 35cfea92
......@@ -15,16 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "VectorSource.h"
#include "db/meta/Meta.h"
#include "db/engine/ExecutionEngine.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -32,20 +31,26 @@ namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids);
Status
Add(const VectorSourcePtr& source, IDNumbers& vector_ids);
size_t GetCurrentMem();
size_t
GetCurrentMem();
size_t GetMemLeft();
size_t
GetMemLeft();
bool IsFull();
bool
IsFull();
Status Serialize();
Status
Serialize();
private:
Status CreateTableFile();
Status
CreateTableFile();
private:
const std::string table_id_;
......@@ -55,10 +60,10 @@ class MemTableFile {
size_t current_mem_;
ExecutionEnginePtr execution_engine_;
}; //MemTableFile
}; // MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/VectorSource.cpp
浏览文件 @ 35cfea92
......@@ -15,35 +15,28 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/VectorSource.h"
#include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h"
#include "utils/Log.h"
#include "db/engine/ExecutionEngine.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace engine {
VectorSource::VectorSource(const size_t &n,
const float *vectors) :
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
VectorSource::VectorSource(const size_t& n, const float* vectors)
: n_(n), vectors_(vectors), id_generator_(std::make_shared<SimpleIDGenerator>()) {
current_num_vectors_added = 0;
}
Status
VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
VectorSource::Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids) {
server::CollectAddMetrics metrics(n_, table_file_schema.dimension_);
num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ?
num_vectors_to_add : n_ - current_num_vectors_added;
num_vectors_added =
current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added;
IDNumbers vector_ids_to_add;
if (vector_ids.empty()) {
id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add);
......@@ -58,8 +51,7 @@ VectorSource::Add(const ExecutionEnginePtr &execution_engine,
vector_ids_to_add.data());
if (status.ok()) {
current_num_vectors_added += num_vectors_added;
vector_ids_.insert(vector_ids_.end(),
std::make_move_iterator(vector_ids_to_add.begin()),
vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()),
std::make_move_iterator(vector_ids_to_add.end()));
} else {
ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString();
......@@ -83,6 +75,6 @@ VectorSource::GetVectorIds() {
return vector_ids_;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/insert/VectorSource.h
浏览文件 @ 35cfea92
......@@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "db/meta/Meta.h"
#include "db/IDGenerator.h"
#include "db/engine/ExecutionEngine.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <memory>
......@@ -31,32 +30,33 @@ namespace engine {
class VectorSource {
public:
VectorSource(const size_t &n, const float *vectors);
VectorSource(const size_t& n, const float* vectors);
Status Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids);
Status
Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids);
size_t GetNumVectorsAdded();
size_t
GetNumVectorsAdded();
bool AllAdded();
bool
AllAdded();
IDNumbers GetVectorIds();
IDNumbers
GetVectorIds();
private:
const size_t n_;
const float *vectors_;
const float* vectors_;
IDNumbers vector_ids_;
size_t current_num_vectors_added;
std::shared_ptr<IDGenerator> id_generator_;
}; //VectorSource
}; // VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/Meta.h
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MetaTypes.h"
......@@ -25,84 +24,104 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
static const char *META_TABLES = "Tables";
static const char *META_TABLEFILES = "TableFiles";
static const char* META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles";
class Meta {
public:
virtual ~Meta() = default;
virtual Status CreateTable(TableSchema &table_schema) = 0;
virtual Status
CreateTable(TableSchema& table_schema) = 0;
virtual Status DescribeTable(TableSchema &table_schema) = 0;
virtual Status
DescribeTable(TableSchema& table_schema) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not) = 0;
virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status
AllTables(std::vector<TableSchema>& table_schema_array) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status DeleteTable(const std::string &table_id) = 0;
virtual Status
DeleteTable(const std::string& table_id) = 0;
virtual Status DeleteTableFiles(const std::string &table_id) = 0;
virtual Status
DeleteTableFiles(const std::string& table_id) = 0;
virtual Status CreateTableFile(TableFileSchema &file_schema) = 0;
virtual Status
CreateTableFile(TableFileSchema& file_schema) = 0;
virtual Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) = 0;
virtual Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0;
virtual Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) = 0;
virtual Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
virtual Status UpdateTableFilesToIndex(const std::string &table_id) = 0;
virtual Status
UpdateTableFilesToIndex(const std::string& table_id) = 0;
virtual Status UpdateTableFile(TableFileSchema &file_schema) = 0;
virtual Status
UpdateTableFile(TableFileSchema& file_schema) = 0;
virtual Status UpdateTableFiles(TableFilesSchema &files) = 0;
virtual Status
UpdateTableFiles(TableFilesSchema& files) = 0;
virtual Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) = 0;
virtual Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) = 0;
virtual Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) = 0;
virtual Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status Archive() = 0;
virtual Status
Archive() = 0;
virtual Status FilesToIndex(TableFilesSchema &) = 0;
virtual Status
FilesToIndex(TableFilesSchema&) = 0;
virtual Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) = 0;
virtual Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status DropTableIndex(const std::string &table_id) = 0;
virtual Status
DropTableIndex(const std::string& table_id) = 0;
virtual Status CleanUp() = 0;
virtual Status
CleanUp() = 0;
virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
virtual Status DropAll() = 0;
virtual Status
DropAll() = 0;
virtual Status Count(const std::string &table_id, uint64_t &result) = 0;
}; // MetaData
virtual Status
Count(const std::string& table_id, uint64_t& result) = 0;
}; // MetaData
using MetaPtr = std::shared_ptr<Meta>;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MetaConsts.h
浏览文件 @ 35cfea92
......@@ -38,7 +38,7 @@ const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7 * D_SEC;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MetaFactory.cpp
浏览文件 @ 35cfea92
......@@ -16,26 +16,26 @@
// under the License.
#include "db/meta/MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
#include "SqliteMetaImpl.h"
#include "db/Utils.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <string.h>
#include <memory>
#include <sstream>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions
MetaFactory::BuildOption(const std::string &path) {
MetaFactory::BuildOption(const std::string& path) {
auto p = path;
if (p == "") {
srand(time(nullptr));
......@@ -51,7 +51,7 @@ MetaFactory::BuildOption(const std::string &path) {
}
meta::MetaPtr
MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
MetaFactory::Build(const DBMetaOptions& metaOptions, const int& mode) {
std::string uri = metaOptions.backend_uri_;
utils::MetaUriInfo uri_info;
......@@ -73,6 +73,6 @@ MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MetaFactory.h
浏览文件 @ 35cfea92
......@@ -28,11 +28,13 @@ namespace engine {
class MetaFactory {
public:
static DBMetaOptions BuildOption(const std::string &path = "");
static DBMetaOptions
BuildOption(const std::string& path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
static meta::MetaPtr
Build(const DBMetaOptions& metaOptions, const int& mode);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MetaTypes.h
浏览文件 @ 35cfea92
......@@ -17,22 +17,22 @@
#pragma once
#include "db/engine/ExecutionEngine.h"
#include "db/Constants.h"
#include "db/engine/ExecutionEngine.h"
#include <vector>
#include <map>
#include <string>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
......@@ -50,7 +50,7 @@ struct TableSchema {
size_t id_ = 0;
std::string table_id_;
int32_t state_ = (int) NORMAL;
int32_t state_ = (int)NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int64_t flag_ = 0;
......@@ -58,7 +58,7 @@ struct TableSchema {
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST;
int32_t metric_type_ = DEFAULT_METRIC_TYPE;
}; // TableSchema
}; // TableSchema
struct TableFileSchema {
typedef enum {
......@@ -83,17 +83,17 @@ struct TableFileSchema {
std::string location_;
int64_t updated_time_ = 0;
int64_t created_on_ = 0;
int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; // not persist to meta
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
}; // TableFileSchema
int32_t nlist_ = DEFAULT_NLIST; // not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta
}; // TableFileSchema
using TableFileSchemaPtr = std::shared_ptr<meta::TableFileSchema>;
using TableFilesSchema = std::vector<TableFileSchema>;
using DatePartionedTableFilesSchema = std::map<DateT, TableFilesSchema>;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MySQLConnectionPool.cpp
浏览文件 @ 35cfea92
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/meta/MySQLConnectionPool.h"
namespace zilliz {
......@@ -28,7 +27,7 @@ namespace meta {
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *
mysqlpp::Connection*
MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
......@@ -40,7 +39,7 @@ MySQLConnectionPool::grab() {
// Other half of in-use conn count limit
void
MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
MySQLConnectionPool::release(const mysqlpp::Connection* pc) {
mysqlpp::ConnectionPool::release(pc);
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
......@@ -64,27 +63,25 @@ MySQLConnectionPool::getDB() {
}
// Superclass overrides
mysqlpp::Connection *
mysqlpp::Connection*
MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
mysqlpp::Connection* conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
conn->connect(db_.empty() ? 0 : db_.c_str(), server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_);
return conn;
} catch (const mysqlpp::ConnectionFailed &er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
} catch (const mysqlpp::ConnectionFailed& er) {
ENGINE_LOG_ERROR << "Failed to connect to database server"
<< ": " << er.what();
return nullptr;
}
}
void
MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
MySQLConnectionPool::destroy(mysqlpp::Connection* cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
......@@ -95,7 +92,7 @@ MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MySQLConnectionPool.h
浏览文件 @ 35cfea92
......@@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include "mysql++/mysql++.h"
#include <string>
#include <unistd.h>
#include <atomic>
#include <string>
#include "utils/Log.h"
......@@ -32,20 +31,16 @@ namespace meta {
class MySQLConnectionPool : public mysqlpp::ConnectionPool {
public:
// The object's only constructor
MySQLConnectionPool(std::string dbName,
std::string userName,
std::string passWord,
std::string serverIp,
int port = 0,
int maxPoolSize = 8) :
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
MySQLConnectionPool(std::string dbName, std::string userName, std::string passWord, std::string serverIp,
int port = 0, int maxPoolSize = 8)
: db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
conns_in_use_ = 0;
max_idle_time_ = 10; //10 seconds
max_idle_time_ = 10; // 10 seconds
}
// The destructor. We _must_ call ConnectionPool::clear() here,
......@@ -54,24 +49,30 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool {
clear();
}
mysqlpp::Connection *grab() override;
mysqlpp::Connection*
grab() override;
// Other half of in-use conn count limit
void release(const mysqlpp::Connection *pc) override;
void
release(const mysqlpp::Connection* pc) override;
// int getConnectionsInUse();
//
// void set_max_idle_time(int max_idle);
// int getConnectionsInUse();
//
// void set_max_idle_time(int max_idle);
std::string getDB();
std::string
getDB();
protected:
// Superclass overrides
mysqlpp::Connection *create() override;
mysqlpp::Connection*
create() override;
void destroy(mysqlpp::Connection *cp) override;
void
destroy(mysqlpp::Connection* cp) override;
unsigned int max_idle_time() override;
unsigned int
max_idle_time() override;
private:
// Number of connections currently in use
......@@ -86,7 +87,7 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool {
unsigned int max_idle_time_;
};
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/db/meta/MySQLMetaImpl.h
浏览文件 @ 35cfea92
......@@ -18,14 +18,14 @@
#pragma once
#include "Meta.h"
#include "db/Options.h"
#include "MySQLConnectionPool.h"
#include "db/Options.h"
#include <mysql++/mysql++.h>
#include <memory>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -34,77 +34,101 @@ namespace meta {
class MySQLMetaImpl : public Meta {
public:
MySQLMetaImpl(const DBMetaOptions &options, const int &mode);
MySQLMetaImpl(const DBMetaOptions& options, const int& mode);
~MySQLMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status
CreateTable(TableSchema& table_schema) override;
Status DescribeTable(TableSchema &table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status DeleteTable(const std::string &table_id) override;
Status
DeleteTable(const std::string& table_id) override;
Status DeleteTableFiles(const std::string &table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
Status CreateTableFile(TableFileSchema &file_schema) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
Status DropPartitionsByDates(const std::string &table_id,
const DatesT &dates) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status DropTableIndex(const std::string &table_id) override;
Status
DropTableIndex(const std::string& table_id) override;
Status UpdateTableFile(TableFileSchema &file_schema) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
Status UpdateTableFilesToIndex(const std::string &table_id) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
Status UpdateTableFiles(TableFilesSchema &files) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) override;
Status FilesToMerge(const std::string &table_id,
DatePartionedTableFilesSchema &files) override;
Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override;
Status FilesToIndex(TableFilesSchema &) override;
Status
FilesToIndex(TableFilesSchema&) override;
Status Archive() override;
Status
Archive() override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
Status CleanUp() override;
Status
CleanUp() override;
Status CleanUpFilesWithTTL(uint16_t seconds) override;
Status
CleanUpFilesWithTTL(uint16_t seconds) override;
Status DropAll() override;
Status
DropAll() override;
Status Count(const std::string &table_id, uint64_t &result) override;
Status
Count(const std::string& table_id, uint64_t& result) override;
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
Status
DiscardFiles(int64_t to_discard_size);
void
ValidateMetaSchema();
Status
Initialize();
private:
const DBMetaOptions options_;
......@@ -113,10 +137,10 @@ class MySQLMetaImpl : public Meta {
std::shared_ptr<MySQLConnectionPool> mysql_connection_pool_;
bool safe_grab_ = false;
// std::mutex connectionMutex_;
}; // DBMetaImpl
// std::mutex connectionMutex_;
}; // DBMetaImpl
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/db/meta/SqliteMetaImpl.h
浏览文件 @ 35cfea92
......@@ -21,8 +21,8 @@
#include "db/Options.h"
#include <mutex>
#include <vector>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
......@@ -30,86 +30,112 @@ namespace engine {
namespace meta {
auto
StoragePrototype(const std::string &path);
StoragePrototype(const std::string& path);
class SqliteMetaImpl : public Meta {
public:
explicit SqliteMetaImpl(const DBMetaOptions &options);
explicit SqliteMetaImpl(const DBMetaOptions& options);
~SqliteMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status
CreateTable(TableSchema& table_schema) override;
Status DescribeTable(TableSchema &table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status DeleteTable(const std::string &table_id) override;
Status
DeleteTable(const std::string& table_id) override;
Status DeleteTableFiles(const std::string &table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
Status CreateTableFile(TableFileSchema &file_schema) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status DropTableIndex(const std::string &table_id) override;
Status
DropTableIndex(const std::string& table_id) override;
Status UpdateTableFilesToIndex(const std::string &table_id) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
Status UpdateTableFile(TableFileSchema &file_schema) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
Status UpdateTableFiles(TableFilesSchema &files) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) override;
Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override;
Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override;
Status FilesToIndex(TableFilesSchema &) override;
Status
FilesToIndex(TableFilesSchema&) override;
Status Archive() override;
Status
Archive() override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
Status CleanUp() override;
Status
CleanUp() override;
Status CleanUpFilesWithTTL(uint16_t seconds) override;
Status
CleanUpFilesWithTTL(uint16_t seconds) override;
Status DropAll() override;
Status
DropAll() override;
Status Count(const std::string &table_id, uint64_t &result) override;
Status
Count(const std::string& table_id, uint64_t& result) override;
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
Status
DiscardFiles(int64_t to_discard_size);
void
ValidateMetaSchema();
Status
Initialize();
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
}; // DBMetaImpl
}; // DBMetaImpl
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
cpp/src/main.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/metrics/MetricBase.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/metrics/Metrics.cpp
浏览文件 @ 35cfea92
......@@ -16,8 +16,8 @@
// under the License.
#include "metrics/Metrics.h"
#include "server/Config.h"
#include "PrometheusMetrics.h"
#include "server/Config.h"
#include <string>
......@@ -25,15 +25,15 @@ namespace zilliz {
namespace milvus {
namespace server {
MetricsBase &
MetricsBase&
Metrics::GetInstance() {
static MetricsBase &instance = CreateMetricsCollector();
static MetricsBase& instance = CreateMetricsCollector();
return instance;
}
MetricsBase &
MetricsBase&
Metrics::CreateMetricsCollector() {
Config &config = Config::GetInstance();
Config& config = Config::GetInstance();
std::string collector_type_str;
config.GetMetricConfigCollector(collector_type_str);
......@@ -45,6 +45,6 @@ Metrics::CreateMetricsCollector() {
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/metrics/Metrics.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/metrics/PrometheusMetrics.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/metrics/SystemInfo.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/metrics/SystemInfo.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Algorithm.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Algorithm.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Definition.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/JobMgr.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/JobMgr.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/ResourceFactory.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/ResourceMgr.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/ResourceMgr.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/SchedInst.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/SchedInst.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Scheduler.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Scheduler.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/TaskCreator.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/TaskCreator.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/TaskTable.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/TaskTable.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Utils.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/Utils.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/action/Action.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/event/Event.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/job/DeleteJob.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/job/DeleteJob.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/job/Job.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/job/SearchJob.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/job/SearchJob.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/resource/Node.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/resource/Node.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/DeleteTask.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/Path.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/SearchTask.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/Task.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/TestTask.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/scheduler/task/TestTask.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/grpc/ClientProxy.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/grpc/ClientProxy.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/grpc/GrpcClient.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/grpc/GrpcClient.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/include/MilvusApi.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/include/Status.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/sdk/interface/Status.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/Config.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/Config.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/DBWrapper.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/DBWrapper.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/Server.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/server/Server.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/BlockingQueue.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/CommonUtil.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/CommonUtil.h 100755 → 100644
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/Error.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/Exception.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/Log.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/LogUtil.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/LogUtil.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/SignalUtil.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/SignalUtil.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/Status.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/Status.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/StringHelpFunctions.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/ThreadPool.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/TimeRecorder.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/TimeRecorder.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/ValidationUtil.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/utils/ValidationUtil.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/wrapper/ConfAdapter.cpp
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/wrapper/ConfAdapter.h
浏览文件 @ 35cfea92
此差异已折叠。
cpp/src/wrapper/VecIndex.cpp
浏览文件 @ 35cfea92
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或