Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
c8bcf53d
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c8bcf53d
编写于
5月 27, 2019
作者:
X
Xu Peng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(db): refactor for db impl
Former-commit-id: f7488d5189ed1c34d9b3e8e3da4eaa71766f9c11
上级
541692c2
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
89 addition
and
93 deletion
+89
-93
cpp/src/db/DBImpl.h
cpp/src/db/DBImpl.h
+22
-22
cpp/src/db/DBImpl.inl
cpp/src/db/DBImpl.inl
+67
-71
未找到文件。
cpp/src/db/DBImpl.h
浏览文件 @
c8bcf53d
...
...
@@ -35,10 +35,10 @@ public:
virtual
Status
add_group
(
meta
::
TableSchema
&
table_schema
)
override
;
virtual
Status
get_group
(
meta
::
TableSchema
&
table_schema
)
override
;
virtual
Status
has_group
(
const
std
::
string
&
table_id
_
,
bool
&
has_or_not_
)
override
;
virtual
Status
has_group
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
virtual
Status
add_vectors
(
const
std
::
string
&
table_id
_
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
_
)
override
;
virtual
Status
add_vectors
(
const
std
::
string
&
table_id
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
override
;
virtual
Status
search
(
const
std
::
string
&
table_id
,
size_t
k
,
size_t
nq
,
const
float
*
vectors
,
QueryResults
&
results
)
override
;
...
...
@@ -54,30 +54,30 @@ public:
private:
void
background_build_i
ndex
();
Status
build_i
ndex
(
const
meta
::
TableFileSchema
&
);
Status
try_build_i
ndex
();
Status
merge_f
iles
(
const
std
::
string
&
table_id
,
void
BackgroundBuildI
ndex
();
Status
BuildI
ndex
(
const
meta
::
TableFileSchema
&
);
Status
TryBuildI
ndex
();
Status
MergeF
iles
(
const
std
::
string
&
table_id
,
const
meta
::
DateT
&
date
,
const
meta
::
TableFilesSchema
&
files
);
Status
background_merge_f
iles
(
const
std
::
string
&
table_id
);
Status
BackgroundMergeF
iles
(
const
std
::
string
&
table_id
);
void
try_schedule_compaction
();
void
start_timer_task
(
int
interval_
);
void
background_timer_task
(
int
interval_
);
void
TrySchedule
();
void
StartTimerTasks
(
int
interval
);
void
BackgroundTimerTask
(
int
interval
);
static
void
BGWork
(
void
*
db
);
void
background_c
all
();
void
background_c
ompaction
();
void
BackgroundC
all
();
void
BackgroundC
ompaction
();
Env
*
const
_env
;
const
Options
_options
;
Env
*
const
env_
;
const
Options
options_
;
std
::
mutex
_mutex
;
std
::
condition_variable
_bg_work_finish_signal
;
bool
_bg_compaction_scheduled
;
Status
_bg_error
;
std
::
atomic
<
bool
>
_shutting_down
;
std
::
mutex
mutex_
;
std
::
condition_variable
bg_work_finish_signal_
;
bool
bg_compaction_scheduled_
;
Status
bg_error_
;
std
::
atomic
<
bool
>
shutting_down_
;
std
::
mutex
build_index_mutex_
;
bool
bg_build_index_started_
;
...
...
@@ -85,8 +85,8 @@ private:
std
::
thread
bg_timer_thread_
;
MetaPtr
_pMeta
;
MemManagerPtr
_pMemMgr
;
MetaPtr
pMeta_
;
MemManagerPtr
pMemMgr_
;
};
// DBImpl
...
...
cpp/src/db/DBImpl.inl
浏览文件 @
c8bcf53d
...
...
@@ -23,35 +23,35 @@ namespace engine {
template<typename EngineT>
DBImpl<EngineT>::DBImpl(const Options& options)
:
_env
(options.env),
_options
(options),
_bg_compaction_scheduled
(false),
_shutting_down
(false),
:
env_
(options.env),
options_
(options),
bg_compaction_scheduled_
(false),
shutting_down_
(false),
bg_build_index_started_(false),
_pMeta(new meta::DBMetaImpl(_options
.meta)),
_pMemMgr(new MemManager<EngineT>(_pMeta, _options
)) {
start_timer_task(_options
.memory_sync_interval);
pMeta_(new meta::DBMetaImpl(options_
.meta)),
pMemMgr_(new MemManager<EngineT>(pMeta_, options_
)) {
StartTimerTasks(options_
.memory_sync_interval);
}
template<typename EngineT>
Status DBImpl<EngineT>::add_group(meta::TableSchema& table_schema) {
return
_pMeta
->CreateTable(table_schema);
return
pMeta_
->CreateTable(table_schema);
}
template<typename EngineT>
Status DBImpl<EngineT>::get_group(meta::TableSchema& table_schema) {
return
_pMeta
->DescribeTable(table_schema);
return
pMeta_
->DescribeTable(table_schema);
}
template<typename EngineT>
Status DBImpl<EngineT>::has_group(const std::string& table_id, bool& has_or_not) {
return
_pMeta
->HasTable(table_id, has_or_not);
return
pMeta_
->HasTable(table_id, has_or_not);
}
template<typename EngineT>
Status DBImpl<EngineT>::add_vectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) {
Status status =
_pMemMgr
->add_vectors(table_id_, n, vectors, vector_ids_);
Status status =
pMemMgr_
->add_vectors(table_id_, n, vectors, vector_ids_);
if (!status.ok()) {
return status;
}
...
...
@@ -69,7 +69,7 @@ Status DBImpl<EngineT>::search(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
meta::DatePartionedTableFilesSchema files;
auto status =
_pMeta
->FilesToSearch(table_id, dates, files);
auto status =
pMeta_
->FilesToSearch(table_id, dates, files);
if (!status.ok()) { return status; }
LOG(DEBUG) << "Search DateT Size=" << files.size();
...
...
@@ -195,59 +195,59 @@ Status DBImpl<EngineT>::search(const std::string& table_id, size_t k, size_t nq,
}
template<typename EngineT>
void DBImpl<EngineT>::
start_timer_task(int interval_
) {
bg_timer_thread_ = std::thread(&DBImpl<EngineT>::
background_timer_task, this, interval_
);
void DBImpl<EngineT>::
StartTimerTasks(int interval
) {
bg_timer_thread_ = std::thread(&DBImpl<EngineT>::
BackgroundTimerTask, this, interval
);
}
template<typename EngineT>
void DBImpl<EngineT>::
background_timer_task(int interval_
) {
void DBImpl<EngineT>::
BackgroundTimerTask(int interval
) {
Status status;
while (true) {
if (!
_bg_error
.ok()) break;
if (
_shutting_down
.load(std::memory_order_acquire)) break;
if (!
bg_error_
.ok()) break;
if (
shutting_down_
.load(std::memory_order_acquire)) break;
std::this_thread::sleep_for(std::chrono::seconds(interval
_
));
std::this_thread::sleep_for(std::chrono::seconds(interval));
try_schedule_compaction
();
TrySchedule
();
}
}
template<typename EngineT>
void DBImpl<EngineT>::
try_schedule_compaction
() {
if (
_bg_compaction_scheduled
) return;
if (!
_bg_error
.ok()) return;
void DBImpl<EngineT>::
TrySchedule
() {
if (
bg_compaction_scheduled_
) return;
if (!
bg_error_
.ok()) return;
_bg_compaction_scheduled
= true;
_env
->Schedule(&DBImpl<EngineT>::BGWork, this);
bg_compaction_scheduled_
= true;
env_
->Schedule(&DBImpl<EngineT>::BGWork, this);
}
template<typename EngineT>
void DBImpl<EngineT>::BGWork(void* db_) {
reinterpret_cast<DBImpl*>(db_)->
background_c
all();
reinterpret_cast<DBImpl*>(db_)->
BackgroundC
all();
}
template<typename EngineT>
void DBImpl<EngineT>::
background_c
all() {
std::lock_guard<std::mutex> lock(
_mutex
);
assert(
_bg_compaction_scheduled
);
void DBImpl<EngineT>::
BackgroundC
all() {
std::lock_guard<std::mutex> lock(
mutex_
);
assert(
bg_compaction_scheduled_
);
if (!
_bg_error.ok() || _shutting_down
.load(std::memory_order_acquire))
if (!
bg_error_.ok() || shutting_down_
.load(std::memory_order_acquire))
return ;
background_c
ompaction();
BackgroundC
ompaction();
_bg_compaction_scheduled
= false;
_bg_work_finish_signal
.notify_all();
bg_compaction_scheduled_
= false;
bg_work_finish_signal_
.notify_all();
}
template<typename EngineT>
Status DBImpl<EngineT>::
merge_f
iles(const std::string& table_id, const meta::DateT& date,
Status DBImpl<EngineT>::
MergeF
iles(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
meta::TableFileSchema table_file;
table_file.table_id = table_id;
table_file.date = date;
Status status =
_pMeta
->CreateTableFile(table_file);
Status status =
pMeta_
->CreateTableFile(table_file);
if (!status.ok()) {
LOG(INFO) << status.ToString() << std::endl;
...
...
@@ -267,19 +267,19 @@ Status DBImpl<EngineT>::merge_files(const std::string& table_id, const meta::Dat
LOG(DEBUG) << "Merging file " << file_schema.file_id;
index_size = index.Size();
if (index_size >=
_options
.index_trigger_size) break;
if (index_size >=
options_
.index_trigger_size) break;
}
index.Serialize();
if (index_size >=
_options
.index_trigger_size) {
if (index_size >=
options_
.index_trigger_size) {
table_file.file_type = meta::TableFileSchema::TO_INDEX;
} else {
table_file.file_type = meta::TableFileSchema::RAW;
}
table_file.size = index_size;
updated.push_back(table_file);
status =
_pMeta
->UpdateTableFiles(updated);
status =
pMeta_
->UpdateTableFiles(updated);
LOG(DEBUG) << "New merged file " << table_file.file_id <<
" of size=" << index.PhysicalSize()/(1024*1024) << " M";
...
...
@@ -289,43 +289,39 @@ Status DBImpl<EngineT>::merge_files(const std::string& table_id, const meta::Dat
}
template<typename EngineT>
Status DBImpl<EngineT>::
background_merge_f
iles(const std::string& table_id) {
Status DBImpl<EngineT>::
BackgroundMergeF
iles(const std::string& table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status =
_pMeta
->FilesToMerge(table_id, raw_files);
auto status =
pMeta_
->FilesToMerge(table_id, raw_files);
if (!status.ok()) {
return status;
}
/* if (raw_files.size() == 0) { */
/* return Status::OK(); */
/* } */
bool has_merge = false;
for (auto& kv : raw_files) {
auto files = kv.second;
if (files.size() <=
_options
.merge_trigger_number) {
if (files.size() <=
options_
.merge_trigger_number) {
continue;
}
has_merge = true;
merge_f
iles(table_id, kv.first, kv.second);
MergeF
iles(table_id, kv.first, kv.second);
}
_pMeta
->Archive();
pMeta_
->Archive();
try_build_i
ndex();
TryBuildI
ndex();
_pMeta
->CleanUpFilesWithTTL(1);
pMeta_
->CleanUpFilesWithTTL(1);
return Status::OK();
}
template<typename EngineT>
Status DBImpl<EngineT>::
build_i
ndex(const meta::TableFileSchema& file) {
Status DBImpl<EngineT>::
BuildI
ndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file;
table_file.table_id = file.table_id;
table_file.date = file.date;
Status status =
_pMeta
->CreateTableFile(table_file);
Status status =
pMeta_
->CreateTableFile(table_file);
if (!status.ok()) {
return status;
}
...
...
@@ -342,30 +338,30 @@ Status DBImpl<EngineT>::build_index(const meta::TableFileSchema& file) {
to_remove.file_type = meta::TableFileSchema::TO_DELETE;
meta::TableFilesSchema update_files = {to_remove, table_file};
_pMeta
->UpdateTableFiles(update_files);
pMeta_
->UpdateTableFiles(update_files);
LOG(DEBUG) << "New index file " << table_file.file_id << " of size "
<< index->PhysicalSize()/(1024*1024) << " M"
<< " from file " << to_remove.file_id;
index->Cache();
_pMeta
->Archive();
pMeta_
->Archive();
return Status::OK();
}
template<typename EngineT>
void DBImpl<EngineT>::
background_build_i
ndex() {
void DBImpl<EngineT>::
BackgroundBuildI
ndex() {
std::lock_guard<std::mutex> lock(build_index_mutex_);
assert(bg_build_index_started_);
meta::TableFilesSchema to_index_files;
_pMeta
->FilesToIndex(to_index_files);
pMeta_
->FilesToIndex(to_index_files);
Status status;
for (auto& file : to_index_files) {
/* LOG(DEBUG) << "Buiding index for " << file.location; */
status =
build_i
ndex(file);
status =
BuildI
ndex(file);
if (!status.ok()) {
_bg_error
= status;
bg_error_
= status;
return;
}
}
...
...
@@ -376,25 +372,25 @@ void DBImpl<EngineT>::background_build_index() {
}
template<typename EngineT>
Status DBImpl<EngineT>::
try_build_i
ndex() {
Status DBImpl<EngineT>::
TryBuildI
ndex() {
if (bg_build_index_started_) return Status::OK();
if (
_shutting_down
.load(std::memory_order_acquire)) return Status::OK();
if (
shutting_down_
.load(std::memory_order_acquire)) return Status::OK();
bg_build_index_started_ = true;
std::thread build_index_task(&DBImpl<EngineT>::
background_build_i
ndex, this);
std::thread build_index_task(&DBImpl<EngineT>::
BackgroundBuildI
ndex, this);
build_index_task.detach();
return Status::OK();
}
template<typename EngineT>
void DBImpl<EngineT>::
background_c
ompaction() {
void DBImpl<EngineT>::
BackgroundC
ompaction() {
std::vector<std::string> table_ids;
_pMemMgr
->serialize(table_ids);
pMemMgr_
->serialize(table_ids);
Status status;
for (auto table_id : table_ids) {
status =
background_merge_f
iles(table_id);
status =
BackgroundMergeF
iles(table_id);
if (!status.ok()) {
_bg_error
= status;
bg_error_
= status;
return;
}
}
...
...
@@ -402,21 +398,21 @@ void DBImpl<EngineT>::background_compaction() {
template<typename EngineT>
Status DBImpl<EngineT>::drop_all() {
return
_pMeta
->DropAll();
return
pMeta_
->DropAll();
}
template<typename EngineT>
Status DBImpl<EngineT>::size(long& result) {
return
_pMeta
->Size(result);
return
pMeta_
->Size(result);
}
template<typename EngineT>
DBImpl<EngineT>::~DBImpl() {
{
std::unique_lock<std::mutex> lock(
_mutex
);
_shutting_down
.store(true, std::memory_order_release);
while (
_bg_compaction_scheduled
) {
_bg_work_finish_signal
.wait(lock);
std::unique_lock<std::mutex> lock(
mutex_
);
shutting_down_
.store(true, std::memory_order_release);
while (
bg_compaction_scheduled_
) {
bg_work_finish_signal_
.wait(lock);
}
}
{
...
...
@@ -427,8 +423,8 @@ DBImpl<EngineT>::~DBImpl() {
}
bg_timer_thread_.join();
std::vector<std::string> ids;
_pMemMgr
->serialize(ids);
_env
->Stop();
pMemMgr_
->serialize(ids);
env_
->Stop();
}
} // namespace engine
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录