Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
e94499cd
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e94499cd
编写于
5月 27, 2019
作者:
X
Xu Peng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(db): change groupschema and groupfileschema to table*
Former-commit-id: fbc69151149faa8ed32547bcdd1a9935dac34070
上级
aa60c628
变更
13
展开全部
显示空白变更内容
内联
并排
Showing
13 changed file
with
212 addition
and
212 deletion
+212
-212
cpp/src/db/DB.h
cpp/src/db/DB.h
+2
-2
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+11
-11
cpp/src/db/DBImpl.h
cpp/src/db/DBImpl.h
+3
-3
cpp/src/db/DBMetaImpl.cpp
cpp/src/db/DBMetaImpl.cpp
+118
-118
cpp/src/db/DBMetaImpl.h
cpp/src/db/DBMetaImpl.h
+7
-7
cpp/src/db/LocalMetaImpl.cpp
cpp/src/db/LocalMetaImpl.cpp
+14
-14
cpp/src/db/LocalMetaImpl.h
cpp/src/db/LocalMetaImpl.h
+9
-9
cpp/src/db/MemManager.cpp
cpp/src/db/MemManager.cpp
+4
-4
cpp/src/db/MemManager.h
cpp/src/db/MemManager.h
+2
-2
cpp/src/db/Meta.h
cpp/src/db/Meta.h
+6
-6
cpp/src/db/MetaTypes.h
cpp/src/db/MetaTypes.h
+7
-7
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+6
-6
cpp/unittest/db/meta_tests.cpp
cpp/unittest/db/meta_tests.cpp
+23
-23
未找到文件。
cpp/src/db/DB.h
浏览文件 @
e94499cd
...
...
@@ -21,8 +21,8 @@ class DB {
public:
static
void
Open
(
const
Options
&
options
,
DB
**
dbptr
);
virtual
Status
add_group
(
meta
::
Group
Schema
&
group_info_
)
=
0
;
virtual
Status
get_group
(
meta
::
Group
Schema
&
group_info_
)
=
0
;
virtual
Status
add_group
(
meta
::
Table
Schema
&
group_info_
)
=
0
;
virtual
Status
get_group
(
meta
::
Table
Schema
&
group_info_
)
=
0
;
virtual
Status
delete_vectors
(
const
std
::
string
&
group_id
,
const
meta
::
DatesT
&
dates
)
=
0
;
virtual
Status
has_group
(
const
std
::
string
&
group_id_
,
bool
&
has_or_not_
)
=
0
;
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
e94499cd
...
...
@@ -35,12 +35,12 @@ DBImpl<EngineT>::DBImpl(const Options& options)
}
template
<
typename
EngineT
>
Status
DBImpl
<
EngineT
>::
add_group
(
meta
::
Group
Schema
&
group_info
)
{
Status
DBImpl
<
EngineT
>::
add_group
(
meta
::
Table
Schema
&
group_info
)
{
return
_pMeta
->
add_group
(
group_info
);
}
template
<
typename
EngineT
>
Status
DBImpl
<
EngineT
>::
get_group
(
meta
::
Group
Schema
&
group_info
)
{
Status
DBImpl
<
EngineT
>::
get_group
(
meta
::
Table
Schema
&
group_info
)
{
return
_pMeta
->
get_group
(
group_info
);
}
...
...
@@ -93,7 +93,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
meta
::
GroupFilesSchema
raw_files
;
for
(
auto
&
day_files
:
files
)
{
for
(
auto
&
file
:
day_files
.
second
)
{
file
.
file_type
==
meta
::
Group
FileSchema
::
INDEX
?
file
.
file_type
==
meta
::
Table
FileSchema
::
INDEX
?
index_files
.
push_back
(
file
)
:
raw_files
.
push_back
(
file
);
}
}
...
...
@@ -259,7 +259,7 @@ void DBImpl<EngineT>::background_call() {
template
<
typename
EngineT
>
Status
DBImpl
<
EngineT
>::
merge_files
(
const
std
::
string
&
group_id
,
const
meta
::
DateT
&
date
,
const
meta
::
GroupFilesSchema
&
files
)
{
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group_id
;
group_file
.
date
=
date
;
Status
status
=
_pMeta
->
add_group_file
(
group_file
);
...
...
@@ -277,7 +277,7 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
for
(
auto
&
file
:
files
)
{
index
.
Merge
(
file
.
location
);
auto
file_schema
=
file
;
file_schema
.
file_type
=
meta
::
Group
FileSchema
::
TO_DELETE
;
file_schema
.
file_type
=
meta
::
Table
FileSchema
::
TO_DELETE
;
updated
.
push_back
(
file_schema
);
LOG
(
DEBUG
)
<<
"Merging file "
<<
file_schema
.
file_id
;
index_size
=
index
.
Size
();
...
...
@@ -288,9 +288,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
index
.
Serialize
();
if
(
index_size
>=
_options
.
index_trigger_size
)
{
group_file
.
file_type
=
meta
::
Group
FileSchema
::
TO_INDEX
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
TO_INDEX
;
}
else
{
group_file
.
file_type
=
meta
::
Group
FileSchema
::
RAW
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
RAW
;
}
group_file
.
size
=
index_size
;
updated
.
push_back
(
group_file
);
...
...
@@ -336,8 +336,8 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
}
template
<
typename
EngineT
>
Status
DBImpl
<
EngineT
>::
build_index
(
const
meta
::
Group
FileSchema
&
file
)
{
meta
::
Group
FileSchema
group_file
;
Status
DBImpl
<
EngineT
>::
build_index
(
const
meta
::
Table
FileSchema
&
file
)
{
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
file
.
group_id
;
group_file
.
date
=
file
.
date
;
Status
status
=
_pMeta
->
add_group_file
(
group_file
);
...
...
@@ -350,11 +350,11 @@ Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) {
to_index
.
Load
();
auto
index
=
to_index
.
BuildIndex
(
group_file
.
location
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
INDEX
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
INDEX
;
group_file
.
size
=
index
->
Size
();
auto
to_remove
=
file
;
to_remove
.
file_type
=
meta
::
Group
FileSchema
::
TO_DELETE
;
to_remove
.
file_type
=
meta
::
Table
FileSchema
::
TO_DELETE
;
meta
::
GroupFilesSchema
update_files
=
{
to_remove
,
group_file
};
_pMeta
->
update_files
(
update_files
);
...
...
cpp/src/db/DBImpl.h
浏览文件 @
e94499cd
...
...
@@ -33,8 +33,8 @@ public:
DBImpl
(
const
Options
&
options
);
virtual
Status
add_group
(
meta
::
Group
Schema
&
group_info
)
override
;
virtual
Status
get_group
(
meta
::
Group
Schema
&
group_info
)
override
;
virtual
Status
add_group
(
meta
::
Table
Schema
&
group_info
)
override
;
virtual
Status
get_group
(
meta
::
Table
Schema
&
group_info
)
override
;
virtual
Status
delete_vectors
(
const
std
::
string
&
group_id
,
const
meta
::
DatesT
&
dates
)
override
;
virtual
Status
has_group
(
const
std
::
string
&
group_id_
,
bool
&
has_or_not_
)
override
;
...
...
@@ -62,7 +62,7 @@ public:
private:
void
background_build_index
();
Status
build_index
(
const
meta
::
Group
FileSchema
&
);
Status
build_index
(
const
meta
::
Table
FileSchema
&
);
Status
try_build_index
();
Status
merge_files
(
const
std
::
string
&
group_id
,
const
meta
::
DateT
&
date
,
...
...
cpp/src/db/DBMetaImpl.cpp
浏览文件 @
e94499cd
此差异已折叠。
点击以展开。
cpp/src/db/DBMetaImpl.h
浏览文件 @
e94499cd
...
...
@@ -19,11 +19,11 @@ class DBMetaImpl : public Meta {
public:
DBMetaImpl
(
const
DBMetaOptions
&
options_
);
virtual
Status
add_group
(
Group
Schema
&
group_info
)
override
;
virtual
Status
get_group
(
Group
Schema
&
group_info_
)
override
;
virtual
Status
add_group
(
Table
Schema
&
group_info
)
override
;
virtual
Status
get_group
(
Table
Schema
&
group_info_
)
override
;
virtual
Status
has_group
(
const
std
::
string
&
group_id_
,
bool
&
has_or_not_
)
override
;
virtual
Status
add_group_file
(
Group
FileSchema
&
group_file_info
)
override
;
virtual
Status
add_group_file
(
Table
FileSchema
&
group_file_info
)
override
;
virtual
Status
delete_group_partitions
(
const
std
::
string
&
group_id
,
const
meta
::
DatesT
&
dates
)
override
;
...
...
@@ -32,8 +32,8 @@ public:
bool
&
has_or_not_
)
override
;
virtual
Status
get_group_file
(
const
std
::
string
&
group_id_
,
const
std
::
string
&
file_id_
,
Group
FileSchema
&
group_file_info_
)
override
;
virtual
Status
update_group_file
(
Group
FileSchema
&
group_file_
)
override
;
Table
FileSchema
&
group_file_info_
)
override
;
virtual
Status
update_group_file
(
Table
FileSchema
&
group_file_
)
override
;
virtual
Status
get_group_files
(
const
std
::
string
&
group_id_
,
const
int
date_delta_
,
...
...
@@ -68,10 +68,10 @@ private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextGroupId
(
std
::
string
&
group_id
);
Status
discard_files_of_size
(
long
to_discard_size
);
Status
get_group_no_lock
(
Group
Schema
&
group_info
);
Status
get_group_no_lock
(
Table
Schema
&
group_info
);
std
::
string
GetGroupPath
(
const
std
::
string
&
group_id
);
std
::
string
GetGroupDatePartitionPath
(
const
std
::
string
&
group_id
,
DateT
&
date
);
void
GetGroupFilePath
(
Group
FileSchema
&
group_file
);
void
GetGroupFilePath
(
Table
FileSchema
&
group_file
);
Status
initialize
();
const
DBMetaOptions
_options
;
...
...
cpp/src/db/LocalMetaImpl.cpp
浏览文件 @
e94499cd
...
...
@@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id
}
std
::
string
LocalMetaImpl
::
GetNextGroupFileLocationByPartition
(
const
std
::
string
&
group_id
,
DateT
&
date
,
Group
FileSchema
::
FILE_TYPE
file_type
)
{
std
::
string
suffix
=
(
file_type
==
Group
FileSchema
::
RAW
)
?
".raw"
:
".index"
;
Table
FileSchema
::
FILE_TYPE
file_type
)
{
std
::
string
suffix
=
(
file_type
==
Table
FileSchema
::
RAW
)
?
".raw"
:
".index"
;
SimpleIDGenerator
g
;
std
::
stringstream
ss
;
ss
<<
GetGroupPath
(
group_id
)
<<
"/"
<<
date
<<
"/"
<<
g
.
getNextIDNumber
()
<<
suffix
;
...
...
@@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return
GetGroupMetaPathByGroupPath
(
GetGroupPath
(
group_id
));
}
Status
LocalMetaImpl
::
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
Group
Schema
&
group_info
)
{
Status
LocalMetaImpl
::
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
Table
Schema
&
group_info
)
{
boost
::
property_tree
::
ptree
ptree
;
boost
::
property_tree
::
read_json
(
path
,
ptree
);
auto
files_cnt
=
ptree
.
get_child
(
"files_cnt"
).
data
();
...
...
@@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem
}
Status
LocalMetaImpl
::
GetGroupMetaInfo
(
const
std
::
string
&
group_id
,
Group
Schema
&
group_info
)
{
Status
LocalMetaImpl
::
GetGroupMetaInfo
(
const
std
::
string
&
group_id
,
Table
Schema
&
group_info
)
{
group_info
.
group_id
=
group_id
;
return
GetGroupMetaInfoByPath
(
GetGroupMetaPath
(
group_id
),
group_info
);
}
...
...
@@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() {
return
Status
::
OK
();
}
Status
LocalMetaImpl
::
add_group
(
Group
Schema
&
group_info
)
{
Status
LocalMetaImpl
::
add_group
(
Table
Schema
&
group_info
)
{
std
::
string
real_gid
;
size_t
id
=
SimpleIDGenerator
().
getNextIDNumber
();
if
(
group_info
.
group_id
==
""
)
{
...
...
@@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) {
return
Status
::
OK
();
}
Status
LocalMetaImpl
::
get_group
(
Group
Schema
&
group_info
)
{
Status
LocalMetaImpl
::
get_group
(
Table
Schema
&
group_info
)
{
bool
group_exist
;
has_group
(
group_info
.
group_id
,
group_exist
);
if
(
!
group_exist
)
{
...
...
@@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return
Status
::
OK
();
}
Status
LocalMetaImpl
::
add_group_file
(
Group
FileSchema
&
group_file_info
)
{
Group
Schema
group_info
;
Status
LocalMetaImpl
::
add_group_file
(
Table
FileSchema
&
group_file_info
)
{
Table
Schema
group_info
;
/* auto status = get_group(group_info); */
/* if (!status.ok()) { */
/* return status; */
...
...
@@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
boost
::
filesystem
::
directory_iterator
end_itr
;
for
(
boost
::
filesystem
::
directory_iterator
itr
(
_options
.
path
);
itr
!=
end_itr
;
++
itr
)
{
auto
group_path
=
itr
->
path
().
string
();
Group
Schema
group_info
;
Table
Schema
group_info
;
GetGroupMetaInfoByPath
(
GetGroupMetaPathByGroupPath
(
group_path
),
group_info
);
for
(
boost
::
filesystem
::
directory_iterator
innerItr
(
group_path
);
innerItr
!=
end_itr
;
++
innerItr
)
{
auto
partition_path
=
innerItr
->
path
().
string
();
...
...
@@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
if
(
suffix
==
"index"
)
continue
;
if
(
INDEX_TRIGGER_SIZE
>=
GetFileSize
(
location
))
continue
;
std
::
cout
<<
"[About to index] "
<<
location
<<
std
::
endl
;
Group
FileSchema
f
;
Table
FileSchema
f
;
f
.
location
=
location
;
/* f.group_id = group_id; */
f
.
dimension
=
group_info
.
dimension
;
...
...
@@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */
/*
Group
Schema group_info; */
/*
Table
Schema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */
...
...
@@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */
/*
Group
FileSchema f; */
/*
Table
FileSchema f; */
/* f.location = location; */
/* f.group_id = group_id; */
/* f.dimension = group_info.dimension; */
...
...
@@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_,
Status
LocalMetaImpl
::
get_group_file
(
const
std
::
string
&
group_id_
,
const
std
::
string
&
file_id_
,
Group
FileSchema
&
group_file_info_
)
{
Table
FileSchema
&
group_file_info_
)
{
//PXU TODO
return
Status
::
OK
();
}
...
...
@@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_,
return
Status
::
OK
();
}
Status
LocalMetaImpl
::
update_group_file
(
Group
FileSchema
&
group_file_
)
{
Status
LocalMetaImpl
::
update_group_file
(
Table
FileSchema
&
group_file_
)
{
//PXU TODO
return
Status
::
OK
();
}
...
...
cpp/src/db/LocalMetaImpl.h
浏览文件 @
e94499cd
...
...
@@ -18,11 +18,11 @@ public:
const
size_t
INDEX_TRIGGER_SIZE
=
1024
*
1024
*
500
;
LocalMetaImpl
(
const
DBMetaOptions
&
options_
);
virtual
Status
add_group
(
Group
Schema
&
group_info_
)
override
;
virtual
Status
get_group
(
Group
Schema
&
group_info_
)
override
;
virtual
Status
add_group
(
Table
Schema
&
group_info_
)
override
;
virtual
Status
get_group
(
Table
Schema
&
group_info_
)
override
;
virtual
Status
has_group
(
const
std
::
string
&
group_id_
,
bool
&
has_or_not_
)
override
;
virtual
Status
add_group_file
(
Group
FileSchema
&
group_file_info
)
override
;
virtual
Status
add_group_file
(
Table
FileSchema
&
group_file_info
)
override
;
/* virtual Status delete_group_partitions(const std::string& group_id, */
/* const meta::DatesT& dates) override; */
...
...
@@ -31,8 +31,8 @@ public:
bool
&
has_or_not_
)
override
;
virtual
Status
get_group_file
(
const
std
::
string
&
group_id_
,
const
std
::
string
&
file_id_
,
Group
FileSchema
&
group_file_info_
)
override
;
virtual
Status
update_group_file
(
Group
FileSchema
&
group_file_
)
override
;
Table
FileSchema
&
group_file_info_
)
override
;
virtual
Status
update_group_file
(
Table
FileSchema
&
group_file_
)
override
;
virtual
Status
get_group_files
(
const
std
::
string
&
group_id_
,
const
int
date_delta_
,
...
...
@@ -59,16 +59,16 @@ public:
private:
Status
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
Group
Schema
&
group_info
);
Status
GetGroupMetaInfoByPath
(
const
std
::
string
&
path
,
Table
Schema
&
group_info
);
std
::
string
GetGroupMetaPathByGroupPath
(
const
std
::
string
&
group_path
);
Status
GetGroupMetaInfo
(
const
std
::
string
&
group_id
,
Group
Schema
&
group_info
);
Status
GetGroupMetaInfo
(
const
std
::
string
&
group_id
,
Table
Schema
&
group_info
);
std
::
string
GetNextGroupFileLocationByPartition
(
const
std
::
string
&
group_id
,
DateT
&
date
,
Group
FileSchema
::
FILE_TYPE
file_type
);
Table
FileSchema
::
FILE_TYPE
file_type
);
std
::
string
GetGroupDatePartitionPath
(
const
std
::
string
&
group_id
,
DateT
&
date
);
std
::
string
GetGroupPath
(
const
std
::
string
&
group_id
);
std
::
string
GetGroupMetaPath
(
const
std
::
string
&
group_id
);
Status
CreateGroupMeta
(
const
Group
Schema
&
group_schema
);
Status
CreateGroupMeta
(
const
Table
Schema
&
group_schema
);
long
GetFileSize
(
const
std
::
string
&
filename
);
Status
initialize
();
...
...
cpp/src/db/MemManager.cpp
浏览文件 @
e94499cd
...
...
@@ -22,7 +22,7 @@ namespace engine {
template
<
typename
EngineT
>
MemVectors
<
EngineT
>::
MemVectors
(
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta_ptr
,
const
meta
::
Group
FileSchema
&
schema
,
const
Options
&
options
)
const
meta
::
Table
FileSchema
&
schema
,
const
Options
&
options
)
:
pMeta_
(
meta_ptr
),
options_
(
options
),
schema_
(
schema
),
...
...
@@ -53,11 +53,11 @@ Status MemVectors<EngineT>::serialize(std::string& group_id) {
pEE_
->
Serialize
();
schema_
.
size
=
size
;
schema_
.
file_type
=
(
size
>=
options_
.
index_trigger_size
)
?
meta
::
GroupFileSchema
::
TO_INDEX
:
meta
::
Group
FileSchema
::
RAW
;
meta
::
TableFileSchema
::
TO_INDEX
:
meta
::
Table
FileSchema
::
RAW
;
auto
status
=
pMeta_
->
update_group_file
(
schema_
);
LOG
(
DEBUG
)
<<
"New "
<<
((
schema_
.
file_type
==
meta
::
Group
FileSchema
::
RAW
)
?
"raw"
:
"to_index"
)
LOG
(
DEBUG
)
<<
"New "
<<
((
schema_
.
file_type
==
meta
::
Table
FileSchema
::
RAW
)
?
"raw"
:
"to_index"
)
<<
" file "
<<
schema_
.
file_id
<<
" of size "
<<
pEE_
->
Size
()
/
meta
::
M
<<
" M"
;
pEE_
->
Cache
();
...
...
@@ -85,7 +85,7 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_grou
return
memIt
->
second
;
}
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group_id
;
auto
status
=
_pMeta
->
add_group_file
(
group_file
);
if
(
!
status
.
ok
())
{
...
...
cpp/src/db/MemManager.h
浏览文件 @
e94499cd
...
...
@@ -31,7 +31,7 @@ public:
typedef
std
::
shared_ptr
<
MemVectors
<
EngineT
>>
Ptr
;
explicit
MemVectors
(
const
std
::
shared_ptr
<
meta
::
Meta
>&
,
const
meta
::
Group
FileSchema
&
,
const
Options
&
);
const
meta
::
Table
FileSchema
&
,
const
Options
&
);
void
add
(
size_t
n_
,
const
float
*
vectors_
,
IDNumbers
&
vector_ids_
);
...
...
@@ -52,7 +52,7 @@ private:
MetaPtr
pMeta_
;
Options
options_
;
meta
::
Group
FileSchema
schema_
;
meta
::
Table
FileSchema
schema_
;
IDGenerator
*
_pIdGenerator
;
EnginePtr
pEE_
;
...
...
cpp/src/db/Meta.h
浏览文件 @
e94499cd
...
...
@@ -20,13 +20,13 @@ namespace meta {
class
Meta
{
public:
typedef
std
::
shared_ptr
<
Meta
>
Ptr
;
using
Ptr
=
std
::
shared_ptr
<
Meta
>
;
virtual
Status
add_group
(
Group
Schema
&
group_info
)
=
0
;
virtual
Status
get_group
(
Group
Schema
&
group_info
)
=
0
;
virtual
Status
add_group
(
Table
Schema
&
group_info
)
=
0
;
virtual
Status
get_group
(
Table
Schema
&
group_info
)
=
0
;
virtual
Status
has_group
(
const
std
::
string
&
group_id_
,
bool
&
has_or_not_
)
=
0
;
virtual
Status
add_group_file
(
Group
FileSchema
&
group_file_info
)
=
0
;
virtual
Status
add_group_file
(
Table
FileSchema
&
group_file_info
)
=
0
;
virtual
Status
delete_group_partitions
(
const
std
::
string
&
group_id
,
const
meta
::
DatesT
&
dates
)
=
0
;
...
...
@@ -35,8 +35,8 @@ public:
bool
&
has_or_not_
)
=
0
;
virtual
Status
get_group_file
(
const
std
::
string
&
group_id_
,
const
std
::
string
&
file_id_
,
Group
FileSchema
&
group_file_info_
)
=
0
;
virtual
Status
update_group_file
(
Group
FileSchema
&
group_file_
)
=
0
;
Table
FileSchema
&
group_file_info_
)
=
0
;
virtual
Status
update_group_file
(
Table
FileSchema
&
group_file_
)
=
0
;
virtual
Status
get_group_files
(
const
std
::
string
&
group_id_
,
const
int
date_delta_
,
...
...
cpp/src/db/MetaTypes.h
浏览文件 @
e94499cd
...
...
@@ -18,16 +18,16 @@ typedef int DateT;
const
DateT
EmptyDate
=
-
1
;
typedef
std
::
vector
<
DateT
>
DatesT
;
struct
Group
Schema
{
struct
Table
Schema
{
size_t
id
;
std
::
string
group_id
;
size_t
files_cnt
=
0
;
uint16_t
dimension
;
std
::
string
location
=
""
;
std
::
string
location
;
long
created_on
;
};
//
Group
Schema
};
//
Table
Schema
struct
Group
FileSchema
{
struct
Table
FileSchema
{
typedef
enum
{
NEW
,
RAW
,
...
...
@@ -43,12 +43,12 @@ struct GroupFileSchema {
size_t
size
;
DateT
date
=
EmptyDate
;
uint16_t
dimension
;
std
::
string
location
=
""
;
std
::
string
location
;
long
updated_time
;
long
created_on
;
};
//
Group
FileSchema
};
//
Table
FileSchema
typedef
std
::
vector
<
Group
FileSchema
>
GroupFilesSchema
;
typedef
std
::
vector
<
Table
FileSchema
>
GroupFilesSchema
;
typedef
std
::
map
<
DateT
,
GroupFilesSchema
>
DatePartionedGroupFilesSchema
;
}
// namespace meta
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
e94499cd
...
...
@@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
static
const
int
group_dim
=
256
;
long
size
;
engine
::
meta
::
Group
Schema
group_info
;
engine
::
meta
::
Table
Schema
group_info
;
group_info
.
dimension
=
group_dim
;
group_info
.
group_id
=
group_name
;
engine
::
Status
stat
=
db_
->
add_group
(
group_info
);
engine
::
meta
::
Group
Schema
group_info_get
;
engine
::
meta
::
Table
Schema
group_info_get
;
group_info_get
.
group_id
=
group_name
;
stat
=
db_
->
get_group
(
group_info_get
);
ASSERT_STATS
(
stat
);
...
...
@@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) {
static
const
std
::
string
group_name
=
"test_group"
;
static
const
int
group_dim
=
256
;
engine
::
meta
::
Group
Schema
group_info
;
engine
::
meta
::
Table
Schema
group_info
;
group_info
.
dimension
=
group_dim
;
group_info
.
group_id
=
group_name
;
engine
::
Status
stat
=
db_
->
add_group
(
group_info
);
engine
::
meta
::
Group
Schema
group_info_get
;
engine
::
meta
::
Table
Schema
group_info_get
;
group_info_get
.
group_id
=
group_name
;
stat
=
db_
->
get_group
(
group_info_get
);
ASSERT_STATS
(
stat
);
...
...
@@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) {
static
const
std
::
string
group_name
=
"test_group"
;
static
const
int
group_dim
=
256
;
engine
::
meta
::
Group
Schema
group_info
;
engine
::
meta
::
Table
Schema
group_info
;
group_info
.
dimension
=
group_dim
;
group_info
.
group_id
=
group_name
;
engine
::
Status
stat
=
db_
->
add_group
(
group_info
);
engine
::
meta
::
Group
Schema
group_info_get
;
engine
::
meta
::
Table
Schema
group_info_get
;
group_info_get
.
group_id
=
group_name
;
stat
=
db_
->
get_group
(
group_info_get
);
ASSERT_STATS
(
stat
);
...
...
cpp/unittest/db/meta_tests.cpp
浏览文件 @
e94499cd
...
...
@@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine;
TEST_F
(
MetaTest
,
GROUP_TEST
)
{
auto
group_id
=
"meta_test_group"
;
meta
::
Group
Schema
group
;
meta
::
Table
Schema
group
;
group
.
group_id
=
group_id
;
auto
status
=
impl_
->
add_group
(
group
);
ASSERT_TRUE
(
status
.
ok
());
...
...
@@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) {
TEST_F
(
MetaTest
,
GROUP_FILE_TEST
)
{
auto
group_id
=
"meta_test_group"
;
meta
::
Group
Schema
group
;
meta
::
Table
Schema
group
;
group
.
group_id
=
group_id
;
auto
status
=
impl_
->
add_group
(
group
);
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group
.
group_id
;
status
=
impl_
->
add_group_file
(
group_file
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_EQ
(
group_file
.
file_type
,
meta
::
Group
FileSchema
::
NEW
);
ASSERT_EQ
(
group_file
.
file_type
,
meta
::
Table
FileSchema
::
NEW
);
auto
file_id
=
group_file
.
file_id
;
auto
new_file_type
=
meta
::
Group
FileSchema
::
INDEX
;
auto
new_file_type
=
meta
::
Table
FileSchema
::
INDEX
;
group_file
.
file_type
=
new_file_type
;
status
=
impl_
->
update_group_file
(
group_file
);
...
...
@@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
status
=
impl_
->
update_group_file
(
group_file
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_EQ
(
group_file
.
date
,
meta
::
Meta
::
GetDateWithDelta
(
-
2
));
ASSERT_FALSE
(
group_file
.
file_type
==
meta
::
Group
FileSchema
::
TO_DELETE
);
ASSERT_FALSE
(
group_file
.
file_type
==
meta
::
Table
FileSchema
::
TO_DELETE
);
dates
.
clear
();
dates
.
push_back
(
group_file
.
date
);
...
...
@@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
ASSERT_TRUE
(
status
.
ok
());
status
=
impl_
->
get_group_file
(
group_file
.
group_id
,
group_file
.
file_id
,
group_file
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_TRUE
(
group_file
.
file_type
==
meta
::
Group
FileSchema
::
TO_DELETE
);
ASSERT_TRUE
(
group_file
.
file_type
==
meta
::
Table
FileSchema
::
TO_DELETE
);
}
TEST_F
(
MetaTest
,
ARCHIVE_TEST_DAYS
)
{
...
...
@@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto
impl
=
meta
::
DBMetaImpl
(
options
);
auto
group_id
=
"meta_test_group"
;
meta
::
Group
Schema
group
;
meta
::
Table
Schema
group
;
group
.
group_id
=
group_id
;
auto
status
=
impl
.
add_group
(
group
);
meta
::
GroupFilesSchema
files
;
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group
.
group_id
;
auto
cnt
=
100
;
...
...
@@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
std
::
vector
<
int
>
days
;
for
(
auto
i
=
0
;
i
<
cnt
;
++
i
)
{
status
=
impl
.
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
NEW
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
NEW
;
int
day
=
rand
()
%
(
days_num
*
2
);
group_file
.
created_on
=
ts
-
day
*
meta
::
D_SEC
*
meta
::
US_PS
-
10000
;
status
=
impl
.
update_group_file
(
group_file
);
...
...
@@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status
=
impl
.
get_group_file
(
file
.
group_id
,
file
.
file_id
,
file
);
ASSERT_TRUE
(
status
.
ok
());
if
(
days
[
i
]
<
days_num
)
{
ASSERT_EQ
(
file
.
file_type
,
meta
::
Group
FileSchema
::
NEW
);
ASSERT_EQ
(
file
.
file_type
,
meta
::
Table
FileSchema
::
NEW
);
}
else
{
ASSERT_EQ
(
file
.
file_type
,
meta
::
Group
FileSchema
::
TO_DELETE
);
ASSERT_EQ
(
file
.
file_type
,
meta
::
Table
FileSchema
::
TO_DELETE
);
}
i
++
;
}
...
...
@@ -148,19 +148,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto
impl
=
meta
::
DBMetaImpl
(
options
);
auto
group_id
=
"meta_test_group"
;
meta
::
Group
Schema
group
;
meta
::
Table
Schema
group
;
group
.
group_id
=
group_id
;
auto
status
=
impl
.
add_group
(
group
);
meta
::
GroupFilesSchema
files
;
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group
.
group_id
;
auto
cnt
=
10
;
auto
each_size
=
2UL
;
for
(
auto
i
=
0
;
i
<
cnt
;
++
i
)
{
status
=
impl
.
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
NEW
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
NEW
;
group_file
.
size
=
each_size
*
meta
::
G
;
status
=
impl
.
update_group_file
(
group_file
);
files
.
push_back
(
group_file
);
...
...
@@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status
=
impl
.
get_group_file
(
file
.
group_id
,
file
.
file_id
,
file
);
ASSERT_TRUE
(
status
.
ok
());
if
(
i
<
5
)
{
ASSERT_TRUE
(
file
.
file_type
==
meta
::
Group
FileSchema
::
TO_DELETE
);
ASSERT_TRUE
(
file
.
file_type
==
meta
::
Table
FileSchema
::
TO_DELETE
);
}
else
{
ASSERT_EQ
(
file
.
file_type
,
meta
::
Group
FileSchema
::
NEW
);
ASSERT_EQ
(
file
.
file_type
,
meta
::
Table
FileSchema
::
NEW
);
}
++
i
;
}
...
...
@@ -186,7 +186,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F
(
MetaTest
,
GROUP_FILES_TEST
)
{
auto
group_id
=
"meta_test_group"
;
meta
::
Group
Schema
group
;
meta
::
Table
Schema
group
;
group
.
group_id
=
group_id
;
auto
status
=
impl_
->
add_group
(
group
);
...
...
@@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
int
to_index_files_cnt
=
6
;
int
index_files_cnt
=
7
;
meta
::
Group
FileSchema
group_file
;
meta
::
Table
FileSchema
group_file
;
group_file
.
group_id
=
group
.
group_id
;
for
(
auto
i
=
0
;
i
<
new_files_cnt
;
++
i
)
{
status
=
impl_
->
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
NEW
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
NEW
;
status
=
impl_
->
update_group_file
(
group_file
);
}
for
(
auto
i
=
0
;
i
<
raw_files_cnt
;
++
i
)
{
status
=
impl_
->
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
RAW
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
RAW
;
status
=
impl_
->
update_group_file
(
group_file
);
}
for
(
auto
i
=
0
;
i
<
to_index_files_cnt
;
++
i
)
{
status
=
impl_
->
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
TO_INDEX
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
TO_INDEX
;
status
=
impl_
->
update_group_file
(
group_file
);
}
for
(
auto
i
=
0
;
i
<
index_files_cnt
;
++
i
)
{
status
=
impl_
->
add_group_file
(
group_file
);
group_file
.
file_type
=
meta
::
Group
FileSchema
::
INDEX
;
group_file
.
file_type
=
meta
::
Table
FileSchema
::
INDEX
;
status
=
impl_
->
update_group_file
(
group_file
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录