Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
3a05719e
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3a05719e
编写于
4月 30, 2019
作者:
X
Xu Peng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(db): disable some print
Former-commit-id: 09fb8ca768ba06af3407df8d9ec307f259650eae
上级
9cb81455
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
23 addition
and
21 deletion
+23
-21
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+18
-16
cpp/src/db/DBMetaImpl.cpp
cpp/src/db/DBMetaImpl.cpp
+5
-5
未找到文件。
cpp/src/db/DBImpl.cpp
浏览文件 @
3a05719e
...
...
@@ -70,7 +70,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq,
auto
status
=
_pMeta
->
files_to_search
(
group_id
,
dates
,
files
);
if
(
!
status
.
ok
())
{
return
status
;
}
LOG
(
DEBUG
)
<<
"Search DateT Size="
<<
files
.
size
();
/* LOG(DEBUG) << "Search DateT Size=" << files.size(); */
meta
::
GroupFilesSchema
index_files
;
meta
::
GroupFilesSchema
raw_files
;
...
...
@@ -112,6 +112,8 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq,
memset
(
output_distence
,
0
,
k
*
nq
*
sizeof
(
float
));
memset
(
output_ids
,
0
,
k
*
nq
*
sizeof
(
long
));
long
search_set_size
=
0
;
auto
search_in_index
=
[
&
](
meta
::
GroupFilesSchema
&
file_vec
)
->
void
{
for
(
auto
&
file
:
file_vec
)
{
auto
index
=
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
file
.
location
);
...
...
@@ -120,8 +122,10 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq,
index
=
read_index
(
file
.
location
.
c_str
());
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
InsertItem
(
file
.
location
,
index
);
}
auto
file_size
=
index
->
dim
*
index
->
ntotal
*
4
/
(
1024
*
1024
);
search_set_size
+=
file_size
;
LOG
(
DEBUG
)
<<
"Search file_type "
<<
file
.
file_type
<<
" Of Size: "
<<
index
->
dim
*
index
->
ntotal
*
4
/
(
1024
*
1024
)
<<
" M"
;
<<
file_size
<<
" M"
;
index
->
search
(
nq
,
vectors
,
k
,
output_distence
,
output_ids
);
cluster
(
output_ids
,
output_distence
);
// cluster to each query
memset
(
output_distence
,
0
,
k
*
nq
*
sizeof
(
float
));
...
...
@@ -147,6 +151,8 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq,
search_in_index
(
raw_files
);
search_in_index
(
index_files
);
LOG
(
DEBUG
)
<<
"Search Overall Set Size="
<<
search_set_size
<<
" M"
;
cluster_topk
();
free
(
output_distence
);
...
...
@@ -230,8 +236,8 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date,
auto
file_schema
=
file
;
file_schema
.
file_type
=
meta
::
GroupFileSchema
::
TO_DELETE
;
updated
.
push_back
(
file_schema
);
LOG
(
DEBUG
)
<<
"About to merge file "
<<
file_schema
.
file_id
<<
" of size="
<<
file_schema
.
rows
;
/* LOG(DEBUG) << "About to merge file " << file_schema.file_id << */
/* " of size=" << file_schema.rows; */
index_size
=
group_file
.
dimension
*
index
->
ntotal
;
if
(
index_size
>=
_options
.
index_trigger_size
)
break
;
...
...
@@ -247,8 +253,8 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date,
group_file
.
rows
=
index_size
;
updated
.
push_back
(
group_file
);
status
=
_pMeta
->
update_files
(
updated
);
LOG
(
DEBUG
)
<<
"New merged file "
<<
group_file
.
file_id
<<
" of size="
<<
group_file
.
rows
;
/* LOG(DEBUG) << "New merged file " << group_file.file_id << */
/* " of size=" << group_file.rows; */
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
InsertItem
(
group_file
.
location
,
std
::
make_shared
<
Index
>
(
index
));
...
...
@@ -305,13 +311,13 @@ Status DBImpl::build_index(const meta::GroupFileSchema& file) {
}
auto
from_index
=
dynamic_cast
<
faiss
::
IndexIDMap
*>
(
to_index
->
data
().
get
());
LOG
(
DEBUG
)
<<
"Preparing build_index for file_id="
<<
file
.
file_id
<<
" with new index_file_id="
<<
group_file
.
file_id
<<
std
::
endl
;
/* LOG(DEBUG) << "Preparing build_index for file_id=" << file.file_id */
/* << " with new index_file_id=" << group_file.file_id << std::endl; */
auto
index
=
pBuilder
->
build_all
(
from_index
->
ntotal
,
dynamic_cast
<
faiss
::
IndexFlat
*>
(
from_index
->
index
)
->
xb
.
data
(),
from_index
->
id_map
.
data
());
LOG
(
DEBUG
)
<<
"Ending build_index for file_id="
<<
file
.
file_id
<<
" with new index_file_id="
<<
group_file
.
file_id
<<
std
::
endl
;
/* LOG(DEBUG) << "Ending build_index for file_id=" << file.file_id */
/* << " with new index_file_id=" << group_file.file_id << std::endl; */
/* std::cout << "raw size=" << from_index->ntotal << " index size=" << index->ntotal << std::endl; */
write_index
(
index
,
group_file
.
location
.
c_str
());
group_file
.
file_type
=
meta
::
GroupFileSchema
::
INDEX
;
...
...
@@ -336,14 +342,14 @@ void DBImpl::background_build_index() {
_pMeta
->
files_to_index
(
to_index_files
);
Status
status
;
for
(
auto
&
file
:
to_index_files
)
{
LOG
(
DEBUG
)
<<
"Buiding index for "
<<
file
.
location
;
/* LOG(DEBUG) << "Buiding index for " << file.location; */
status
=
build_index
(
file
);
if
(
!
status
.
ok
())
{
_bg_error
=
status
;
return
;
}
}
LOG
(
DEBUG
)
<<
"All Buiding index Done"
;
/* LOG(DEBUG) << "All Buiding index Done"; */
bg_build_index_started_
=
false
;
bg_build_index_finish_signal_
.
notify_all
();
...
...
@@ -382,21 +388,17 @@ Status DBImpl::count(const std::string& group_id, long& result) {
DBImpl
::~
DBImpl
()
{
{
LOG
(
DEBUG
)
<<
"Start wait background merge thread"
;
std
::
unique_lock
<
std
::
mutex
>
lock
(
_mutex
);
_shutting_down
.
store
(
true
,
std
::
memory_order_release
);
while
(
_bg_compaction_scheduled
)
{
_bg_work_finish_signal
.
wait
(
lock
);
}
LOG
(
DEBUG
)
<<
"Stop wait background merge thread"
;
}
{
LOG
(
DEBUG
)
<<
"Start wait background build index thread"
;
std
::
unique_lock
<
std
::
mutex
>
lock
(
build_index_mutex_
);
while
(
bg_build_index_started_
)
{
bg_build_index_finish_signal_
.
wait
(
lock
);
}
LOG
(
DEBUG
)
<<
"Stop wait background build index thread"
;
}
std
::
vector
<
std
::
string
>
ids
;
_pMemMgr
->
serialize
(
ids
);
...
...
cpp/src/db/DBMetaImpl.cpp
浏览文件 @
3a05719e
#include <sys/stat.h>
#include <unistd.h>
#include <sstream>
#include <iostream>
...
...
@@ -109,7 +109,7 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) {
try
{
auto
id
=
ConnectorPtr
->
insert
(
group_info
);
group_info
.
id
=
id
;
LOG
(
DEBUG
)
<<
"Add group "
<<
id
;
/* LOG(DEBUG) << "Add group " << id; */
}
catch
(...)
{
return
Status
::
DBTransactionError
(
"Add Group Error"
);
}
...
...
@@ -193,7 +193,7 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) {
try
{
auto
id
=
ConnectorPtr
->
insert
(
group_file
);
group_file
.
id
=
id
;
LOG
(
DEBUG
)
<<
"Add group_file of file_id="
<<
group_file
.
file_id
;
/* LOG(DEBUG) << "Add group_file of file_id=" << group_file.file_id; */
}
catch
(...)
{
return
Status
::
DBTransactionError
(
"Add file Error"
);
}
...
...
@@ -434,7 +434,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
boost
::
filesystem
::
remove
(
group_file
.
location
);
}
ConnectorPtr
->
remove
<
GroupFileSchema
>
(
group_file
.
id
);
LOG
(
DEBUG
)
<<
"Removing deleted id="
<<
group_file
.
id
<<
" location="
<<
group_file
.
location
<<
std
::
endl
;
/* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */
}
}
catch
(
std
::
exception
&
e
)
{
LOG
(
DEBUG
)
<<
e
.
what
();
...
...
@@ -470,7 +470,7 @@ Status DBMetaImpl::cleanup() {
boost
::
filesystem
::
remove
(
group_file
.
location
);
}
ConnectorPtr
->
remove
<
GroupFileSchema
>
(
group_file
.
id
);
LOG
(
DEBUG
)
<<
"Removing id="
<<
group_file
.
id
<<
" location="
<<
group_file
.
location
<<
std
::
endl
;
/* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */
}
}
catch
(
std
::
exception
&
e
)
{
LOG
(
DEBUG
)
<<
e
.
what
();
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录