Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
4ce28a1e
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4ce28a1e
编写于
4月 24, 2019
作者:
G
groot
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'jinhai' of 192.168.1.105:jinhai/vecwise_engine into jinhai
Former-commit-id: 6ef6cb1953310c4e01b1787e1a041c25f7dfd951
上级
dc3e4f58
7509e28d
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
76 addition
and
20 deletion
+76
-20
cpp/src/db/DB.h
cpp/src/db/DB.h
+2
-0
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+8
-6
cpp/src/db/DBImpl.h
cpp/src/db/DBImpl.h
+2
-0
cpp/src/db/DBMetaImpl.cpp
cpp/src/db/DBMetaImpl.cpp
+7
-0
cpp/src/db/DBMetaImpl.h
cpp/src/db/DBMetaImpl.h
+2
-0
cpp/src/db/LocalMetaImpl.cpp
cpp/src/db/LocalMetaImpl.cpp
+5
-0
cpp/src/db/LocalMetaImpl.h
cpp/src/db/LocalMetaImpl.h
+2
-0
cpp/src/db/Meta.h
cpp/src/db/Meta.h
+2
-0
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+46
-14
未找到文件。
cpp/src/db/DB.h
浏览文件 @
4ce28a1e
...
@@ -30,6 +30,8 @@ public:
...
@@ -30,6 +30,8 @@ public:
virtual
Status
search
(
const
std
::
string
&
group_id
,
size_t
k
,
size_t
nq
,
virtual
Status
search
(
const
std
::
string
&
group_id
,
size_t
k
,
size_t
nq
,
const
float
*
vectors
,
QueryResults
&
results
)
=
0
;
const
float
*
vectors
,
QueryResults
&
results
)
=
0
;
virtual
Status
drop_all
()
=
0
;
DB
()
=
default
;
DB
()
=
default
;
DB
(
const
DB
&
)
=
delete
;
DB
(
const
DB
&
)
=
delete
;
DB
&
operator
=
(
const
DB
&
)
=
delete
;
DB
&
operator
=
(
const
DB
&
)
=
delete
;
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
4ce28a1e
...
@@ -246,9 +246,9 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
...
@@ -246,9 +246,9 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
return
status
;
return
status
;
}
}
if
(
raw_files
.
size
()
==
0
)
{
/* if (raw_files.size() == 0) { */
return
Status
::
OK
();
/* return Status::OK(); */
}
/* } */
bool
has_merge
=
false
;
bool
has_merge
=
false
;
...
@@ -261,9 +261,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
...
@@ -261,9 +261,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
merge_files
(
group_id
,
kv
.
first
,
kv
.
second
);
merge_files
(
group_id
,
kv
.
first
,
kv
.
second
);
}
}
if
(
has_merge
)
{
try_build_index
();
try_build_index
();
}
_pMeta
->
cleanup_ttl_files
(
1
);
_pMeta
->
cleanup_ttl_files
(
1
);
...
@@ -346,6 +344,10 @@ void DBImpl::background_compaction() {
...
@@ -346,6 +344,10 @@ void DBImpl::background_compaction() {
}
}
}
}
Status
DBImpl
::
drop_all
()
{
return
_pMeta
->
drop_all
();
}
DBImpl
::~
DBImpl
()
{
DBImpl
::~
DBImpl
()
{
{
{
std
::
unique_lock
<
std
::
mutex
>
lock
(
_mutex
);
std
::
unique_lock
<
std
::
mutex
>
lock
(
_mutex
);
...
...
cpp/src/db/DBImpl.h
浏览文件 @
4ce28a1e
...
@@ -36,6 +36,8 @@ public:
...
@@ -36,6 +36,8 @@ public:
virtual
Status
search
(
const
std
::
string
&
group_id
,
size_t
k
,
size_t
nq
,
virtual
Status
search
(
const
std
::
string
&
group_id
,
size_t
k
,
size_t
nq
,
const
float
*
vectors
,
QueryResults
&
results
)
override
;
const
float
*
vectors
,
QueryResults
&
results
)
override
;
virtual
Status
drop_all
()
override
;
virtual
~
DBImpl
();
virtual
~
DBImpl
();
private:
private:
...
...
cpp/src/db/DBMetaImpl.cpp
浏览文件 @
4ce28a1e
...
@@ -427,6 +427,13 @@ Status DBMetaImpl::cleanup() {
...
@@ -427,6 +427,13 @@ Status DBMetaImpl::cleanup() {
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
DBMetaImpl
::
drop_all
()
{
if
(
boost
::
filesystem
::
is_directory
(
_options
.
path
))
{
boost
::
filesystem
::
remove_all
(
_options
.
path
);
}
return
Status
::
OK
();
}
DBMetaImpl
::~
DBMetaImpl
()
{
DBMetaImpl
::~
DBMetaImpl
()
{
cleanup
();
cleanup
();
}
}
...
...
cpp/src/db/DBMetaImpl.h
浏览文件 @
4ce28a1e
...
@@ -48,6 +48,8 @@ public:
...
@@ -48,6 +48,8 @@ public:
virtual
Status
cleanup_ttl_files
(
uint16_t
seconds
)
override
;
virtual
Status
cleanup_ttl_files
(
uint16_t
seconds
)
override
;
virtual
Status
drop_all
()
override
;
virtual
~
DBMetaImpl
();
virtual
~
DBMetaImpl
();
private:
private:
...
...
cpp/src/db/LocalMetaImpl.cpp
浏览文件 @
4ce28a1e
...
@@ -246,6 +246,11 @@ Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
...
@@ -246,6 +246,11 @@ Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
LocalMetaImpl
::
drop_all
()
{
// PXU TODO
return
Status
::
OK
();
}
}
// namespace meta
}
// namespace meta
}
// namespace engine
}
// namespace engine
}
// namespace vecwise
}
// namespace vecwise
...
...
cpp/src/db/LocalMetaImpl.h
浏览文件 @
4ce28a1e
...
@@ -43,6 +43,8 @@ public:
...
@@ -43,6 +43,8 @@ public:
virtual
Status
cleanup_ttl_files
(
uint16_t
seconds
)
override
;
virtual
Status
cleanup_ttl_files
(
uint16_t
seconds
)
override
;
virtual
Status
drop_all
()
override
;
private:
private:
Status
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
GroupSchema
&
group_info
);
Status
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
GroupSchema
&
group_info
);
...
...
cpp/src/db/Meta.h
浏览文件 @
4ce28a1e
...
@@ -83,6 +83,8 @@ public:
...
@@ -83,6 +83,8 @@ public:
virtual
Status
cleanup
()
=
0
;
virtual
Status
cleanup
()
=
0
;
virtual
Status
cleanup_ttl_files
(
uint16_t
)
=
0
;
virtual
Status
cleanup_ttl_files
(
uint16_t
)
=
0
;
virtual
Status
drop_all
()
=
0
;
static
DateT
GetDate
(
const
std
::
time_t
&
t
);
static
DateT
GetDate
(
const
std
::
time_t
&
t
);
static
DateT
GetDate
();
static
DateT
GetDate
();
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
4ce28a1e
...
@@ -7,6 +7,8 @@
...
@@ -7,6 +7,8 @@
#include <faiss/IndexFlat.h>
#include <faiss/IndexFlat.h>
#include <faiss/MetaIndexes.h>
#include <faiss/MetaIndexes.h>
#include <faiss/AutoTune.h>
#include <faiss/AutoTune.h>
#include <thread>
#include <easylogging++.h>
#include "db/DB.h"
#include "db/DB.h"
#include "faiss/Index.h"
#include "faiss/Index.h"
...
@@ -27,6 +29,7 @@ TEST(DBTest, DB_TEST) {
...
@@ -27,6 +29,7 @@ TEST(DBTest, DB_TEST) {
static
const
int
group_dim
=
256
;
static
const
int
group_dim
=
256
;
engine
::
Options
opt
;
engine
::
Options
opt
;
opt
.
memory_sync_interval
=
1
;
opt
.
meta
.
backend_uri
=
"http://127.0.0.1"
;
opt
.
meta
.
backend_uri
=
"http://127.0.0.1"
;
opt
.
meta
.
path
=
"/tmp/vecwise_test/db_test"
;
opt
.
meta
.
path
=
"/tmp/vecwise_test/db_test"
;
...
@@ -38,7 +41,6 @@ TEST(DBTest, DB_TEST) {
...
@@ -38,7 +41,6 @@ TEST(DBTest, DB_TEST) {
group_info
.
dimension
=
group_dim
;
group_info
.
dimension
=
group_dim
;
group_info
.
group_id
=
group_name
;
group_info
.
group_id
=
group_name
;
engine
::
Status
stat
=
db
->
add_group
(
group_info
);
engine
::
Status
stat
=
db
->
add_group
(
group_info
);
//ASSERT_STATS(stat);
engine
::
meta
::
GroupSchema
group_info_get
;
engine
::
meta
::
GroupSchema
group_info_get
;
group_info_get
.
group_id
=
group_name
;
group_info_get
.
group_id
=
group_name
;
...
@@ -47,21 +49,48 @@ TEST(DBTest, DB_TEST) {
...
@@ -47,21 +49,48 @@ TEST(DBTest, DB_TEST) {
ASSERT_EQ
(
group_info_get
.
dimension
,
group_dim
);
ASSERT_EQ
(
group_info_get
.
dimension
,
group_dim
);
engine
::
IDNumbers
vector_ids
;
engine
::
IDNumbers
vector_ids
;
std
::
vector
<
float
>
vec_f
;
engine
::
IDNumbers
target_ids
;
for
(
int
i
=
0
;
i
<
group_dim
;
i
++
){
vec_f
.
push_back
((
float
)
i
);
int
d
=
256
;
vector_ids
.
push_back
(
i
+
1
);
int
nb
=
10
;
float
*
xb
=
new
float
[
d
*
nb
];
for
(
int
i
=
0
;
i
<
nb
;
i
++
)
{
for
(
int
j
=
0
;
j
<
d
;
j
++
)
xb
[
d
*
i
+
j
]
=
drand48
();
xb
[
d
*
i
]
+=
i
/
2000.
;
}
}
stat
=
db
->
add_vectors
(
group_name
,
1
,
vec_f
.
data
(),
vector_ids
);
ASSERT_STATS
(
stat
);
//engine::QueryResults results;
int
qb
=
1
;
//std::vector<float> vec_s = vec_f;
float
*
qxb
=
new
float
[
d
*
qb
];
//stat = db->search(group_name, 1, 1, vec_f.data(), results);
for
(
int
i
=
0
;
i
<
qb
;
i
++
)
{
//ASSERT_STATS(stat);
for
(
int
j
=
0
;
j
<
d
;
j
++
)
qxb
[
d
*
i
+
j
]
=
drand48
();
//ASSERT_EQ(results.size(), 1);
qxb
[
d
*
i
]
+=
i
/
2000.
;
//ASSERT_EQ(results[0][0], vector_ids[0]);
}
int
loop
=
500000
;
for
(
auto
i
=
0
;
i
<
loop
;
++
i
)
{
if
(
i
==
40
)
{
db
->
add_vectors
(
group_name
,
qb
,
qxb
,
target_ids
);
}
else
{
db
->
add_vectors
(
group_name
,
nb
,
xb
,
vector_ids
);
}
}
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
seconds
(
2
));
engine
::
QueryResults
results
;
int
k
=
10
;
LOG
(
DEBUG
)
<<
"PRE"
;
stat
=
db
->
search
(
group_name
,
k
,
qb
,
qxb
,
results
);
LOG
(
DEBUG
)
<<
"POST"
;
ASSERT_STATS
(
stat
);
ASSERT_EQ
(
results
[
0
][
0
],
target_ids
[
0
]);
delete
[]
xb
;
delete
[]
qxb
;
delete
db
;
engine
::
DB
::
Open
(
opt
,
&
db
);
db
->
drop_all
();
delete
db
;
delete
db
;
}
}
...
@@ -141,4 +170,7 @@ TEST(SearchTest, DB_TEST) {
...
@@ -141,4 +170,7 @@ TEST(SearchTest, DB_TEST) {
// TODO(linxj): add groundTruth assert
// TODO(linxj): add groundTruth assert
delete
db
;
delete
db
;
}
engine
::
DB
::
Open
(
opt
,
&
db
);
\ No newline at end of file
db
->
drop_all
();
delete
db
;
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录