Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
2eff64de
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2eff64de
编写于
5月 27, 2019
作者:
X
Xu Peng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(db): refactor for DBMetaImpl
Former-commit-id: 9ced33145335090265c53344b4ff46a977eec356
上级
ca33fbbe
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
46 addition
and
46 deletion
+46
-46
cpp/src/db/DBMetaImpl.cpp
cpp/src/db/DBMetaImpl.cpp
+38
-38
cpp/src/db/DBMetaImpl.h
cpp/src/db/DBMetaImpl.h
+8
-8
未找到文件。
cpp/src/db/DBMetaImpl.cpp
浏览文件 @
2eff64de
...
...
@@ -27,13 +27,13 @@ using namespace sqlite_orm;
inline
auto
StoragePrototype
(
const
std
::
string
&
path
)
{
return
make_storage
(
path
,
make_table
(
"
Group
"
,
make_table
(
"
Table
"
,
make_column
(
"id"
,
&
TableSchema
::
id
,
primary_key
()),
make_column
(
"table_id"
,
&
TableSchema
::
table_id
,
unique
()),
make_column
(
"dimension"
,
&
TableSchema
::
dimension
),
make_column
(
"created_on"
,
&
TableSchema
::
created_on
),
make_column
(
"files_cnt"
,
&
TableSchema
::
files_cnt
,
default_value
(
0
))),
make_table
(
"
Group
File"
,
make_table
(
"
Table
File"
,
make_column
(
"id"
,
&
TableFileSchema
::
id
,
primary_key
()),
make_column
(
"table_id"
,
&
TableFileSchema
::
table_id
),
make_column
(
"file_id"
,
&
TableFileSchema
::
file_id
),
...
...
@@ -49,27 +49,27 @@ inline auto StoragePrototype(const std::string& path) {
using
ConnectorT
=
decltype
(
StoragePrototype
(
""
));
static
std
::
unique_ptr
<
ConnectorT
>
ConnectorPtr
;
std
::
string
DBMetaImpl
::
Get
Group
Path
(
const
std
::
string
&
table_id
)
{
return
_options
.
path
+
"/tables/"
+
table_id
;
std
::
string
DBMetaImpl
::
Get
Table
Path
(
const
std
::
string
&
table_id
)
{
return
options_
.
path
+
"/tables/"
+
table_id
;
}
std
::
string
DBMetaImpl
::
Get
Group
DatePartitionPath
(
const
std
::
string
&
table_id
,
DateT
&
date
)
{
std
::
string
DBMetaImpl
::
Get
Table
DatePartitionPath
(
const
std
::
string
&
table_id
,
DateT
&
date
)
{
std
::
stringstream
ss
;
ss
<<
Get
Group
Path
(
table_id
)
<<
"/"
<<
date
;
ss
<<
Get
Table
Path
(
table_id
)
<<
"/"
<<
date
;
return
ss
.
str
();
}
void
DBMetaImpl
::
Get
Group
FilePath
(
TableFileSchema
&
group_file
)
{
void
DBMetaImpl
::
Get
Table
FilePath
(
TableFileSchema
&
group_file
)
{
if
(
group_file
.
date
==
EmptyDate
)
{
group_file
.
date
=
Meta
::
GetDate
();
}
std
::
stringstream
ss
;
ss
<<
Get
Group
DatePartitionPath
(
group_file
.
table_id
,
group_file
.
date
)
ss
<<
Get
Table
DatePartitionPath
(
group_file
.
table_id
,
group_file
.
date
)
<<
"/"
<<
group_file
.
file_id
;
group_file
.
location
=
ss
.
str
();
}
Status
DBMetaImpl
::
Next
Group
Id
(
std
::
string
&
table_id
)
{
Status
DBMetaImpl
::
Next
Table
Id
(
std
::
string
&
table_id
)
{
std
::
stringstream
ss
;
SimpleIDGenerator
g
;
ss
<<
g
.
getNextIDNumber
();
...
...
@@ -86,20 +86,20 @@ Status DBMetaImpl::NextFileId(std::string& file_id) {
}
DBMetaImpl
::
DBMetaImpl
(
const
DBMetaOptions
&
options_
)
:
_options
(
options_
)
{
i
nitialize
();
:
options_
(
options_
)
{
I
nitialize
();
}
Status
DBMetaImpl
::
i
nitialize
()
{
if
(
!
boost
::
filesystem
::
is_directory
(
_options
.
path
))
{
auto
ret
=
boost
::
filesystem
::
create_directory
(
_options
.
path
);
Status
DBMetaImpl
::
I
nitialize
()
{
if
(
!
boost
::
filesystem
::
is_directory
(
options_
.
path
))
{
auto
ret
=
boost
::
filesystem
::
create_directory
(
options_
.
path
);
if
(
!
ret
)
{
LOG
(
ERROR
)
<<
"Create directory "
<<
_options
.
path
<<
" Error"
;
LOG
(
ERROR
)
<<
"Create directory "
<<
options_
.
path
<<
" Error"
;
}
assert
(
ret
);
}
ConnectorPtr
=
std
::
make_unique
<
ConnectorT
>
(
StoragePrototype
(
_options
.
path
+
"/meta.sqlite"
));
ConnectorPtr
=
std
::
make_unique
<
ConnectorT
>
(
StoragePrototype
(
options_
.
path
+
"/meta.sqlite"
));
ConnectorPtr
->
sync_schema
();
ConnectorPtr
->
open_forever
();
// thread safe option
...
...
@@ -150,7 +150,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string& table_id,
Status
DBMetaImpl
::
CreateTable
(
TableSchema
&
table_schema
)
{
if
(
table_schema
.
table_id
==
""
)
{
Next
Group
Id
(
table_schema
.
table_id
);
Next
Table
Id
(
table_schema
.
table_id
);
}
table_schema
.
files_cnt
=
0
;
table_schema
.
id
=
-
1
;
...
...
@@ -161,11 +161,11 @@ Status DBMetaImpl::CreateTable(TableSchema& table_schema) {
auto
id
=
ConnectorPtr
->
insert
(
table_schema
);
table_schema
.
id
=
id
;
}
catch
(...)
{
return
Status
::
DBTransactionError
(
"Add
Group
Error"
);
return
Status
::
DBTransactionError
(
"Add
Table
Error"
);
}
}
auto
group_path
=
Get
Group
Path
(
table_schema
.
table_id
);
auto
group_path
=
Get
Table
Path
(
table_schema
.
table_id
);
if
(
!
boost
::
filesystem
::
is_directory
(
group_path
))
{
auto
ret
=
boost
::
filesystem
::
create_directories
(
group_path
);
...
...
@@ -191,7 +191,7 @@ Status DBMetaImpl::DescribeTable(TableSchema& table_schema) {
table_schema
.
files_cnt
=
std
::
get
<
2
>
(
groups
[
0
]);
table_schema
.
dimension
=
std
::
get
<
3
>
(
groups
[
0
]);
}
else
{
return
Status
::
NotFound
(
"
Group
"
+
table_schema
.
table_id
+
" not found"
);
return
Status
::
NotFound
(
"
Table
"
+
table_schema
.
table_id
+
" not found"
);
}
}
catch
(
std
::
exception
&
e
)
{
LOG
(
DEBUG
)
<<
e
.
what
();
...
...
@@ -235,7 +235,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) {
file_schema
.
size
=
0
;
file_schema
.
created_on
=
utils
::
GetMicroSecTimeStamp
();
file_schema
.
updated_time
=
file_schema
.
created_on
;
Get
Group
FilePath
(
file_schema
);
Get
Table
FilePath
(
file_schema
);
{
try
{
...
...
@@ -246,7 +246,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) {
}
}
auto
partition_path
=
Get
Group
DatePartitionPath
(
file_schema
.
table_id
,
file_schema
.
date
);
auto
partition_path
=
Get
Table
DatePartitionPath
(
file_schema
.
table_id
,
file_schema
.
date
);
if
(
!
boost
::
filesystem
::
is_directory
(
partition_path
))
{
auto
ret
=
boost
::
filesystem
::
create_directory
(
partition_path
);
...
...
@@ -281,7 +281,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema& files) {
table_file
.
file_type
=
std
::
get
<
3
>
(
file
);
table_file
.
size
=
std
::
get
<
4
>
(
file
);
table_file
.
date
=
std
::
get
<
5
>
(
file
);
Get
Group
FilePath
(
table_file
);
Get
Table
FilePath
(
table_file
);
auto
groupItr
=
groups
.
find
(
table_file
.
table_id
);
if
(
groupItr
==
groups
.
end
())
{
TableSchema
table_schema
;
...
...
@@ -340,7 +340,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
table_file
.
size
=
std
::
get
<
4
>
(
file
);
table_file
.
date
=
std
::
get
<
5
>
(
file
);
table_file
.
dimension
=
table_schema
.
dimension
;
Get
Group
FilePath
(
table_file
);
Get
Table
FilePath
(
table_file
);
auto
dateItr
=
files
.
find
(
table_file
.
date
);
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date
]
=
TableFilesSchema
();
...
...
@@ -385,7 +385,7 @@ Status DBMetaImpl::FilesToMerge(const std::string& table_id,
table_file
.
size
=
std
::
get
<
4
>
(
file
);
table_file
.
date
=
std
::
get
<
5
>
(
file
);
table_file
.
dimension
=
table_schema
.
dimension
;
Get
Group
FilePath
(
table_file
);
Get
Table
FilePath
(
table_file
);
auto
dateItr
=
files
.
find
(
table_file
.
date
);
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date
]
=
TableFilesSchema
();
...
...
@@ -434,7 +434,7 @@ Status DBMetaImpl::GetTableFile(TableFileSchema& file_schema) {
// PXU TODO: Support Swap
Status
DBMetaImpl
::
Archive
()
{
auto
&
criterias
=
_options
.
archive_conf
.
GetCriterias
();
auto
&
criterias
=
options_
.
archive_conf
.
GetCriterias
();
if
(
criterias
.
size
()
==
0
)
{
return
Status
::
OK
();
}
...
...
@@ -464,9 +464,8 @@ Status DBMetaImpl::Archive() {
long
sum
=
0
;
Size
(
sum
);
// PXU TODO: refactor size
auto
to_delete
=
(
sum
-
limit
*
G
);
discard_files_of_size
(
to_delete
);
DiscardFiles
(
to_delete
);
}
}
...
...
@@ -495,17 +494,18 @@ Status DBMetaImpl::Size(long& result) {
return
Status
::
OK
();
}
Status
DBMetaImpl
::
discard_files_of_size
(
long
to_discard_size
)
{
Status
DBMetaImpl
::
DiscardFiles
(
long
to_discard_size
)
{
LOG
(
DEBUG
)
<<
"About to discard size="
<<
to_discard_size
;
if
(
to_discard_size
<=
0
)
{
return
Status
::
OK
();
}
try
{
auto
selected
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
id
,
&
TableFileSchema
::
size
),
where
(
c
(
&
TableFileSchema
::
file_type
)
!=
(
int
)
TableFileSchema
::
TO_DELETE
),
order_by
(
&
TableFileSchema
::
id
),
limit
(
10
));
&
TableFileSchema
::
size
),
where
(
c
(
&
TableFileSchema
::
file_type
)
!=
(
int
)
TableFileSchema
::
TO_DELETE
),
order_by
(
&
TableFileSchema
::
id
),
limit
(
10
));
std
::
vector
<
int
>
ids
;
TableFileSchema
table_file
;
...
...
@@ -536,7 +536,7 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
}
return
discard_files_of_size
(
to_discard_size
);
return
DiscardFiles
(
to_discard_size
);
}
Status
DBMetaImpl
::
UpdateTableFile
(
TableFileSchema
&
file_schema
)
{
...
...
@@ -592,7 +592,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
table_file
.
file_type
=
std
::
get
<
3
>
(
file
);
table_file
.
size
=
std
::
get
<
4
>
(
file
);
table_file
.
date
=
std
::
get
<
5
>
(
file
);
Get
Group
FilePath
(
table_file
);
Get
Table
FilePath
(
table_file
);
if
(
table_file
.
file_type
==
TableFileSchema
::
TO_DELETE
)
{
boost
::
filesystem
::
remove
(
table_file
.
location
);
}
...
...
@@ -628,7 +628,7 @@ Status DBMetaImpl::CleanUp() {
table_file
.
file_type
=
std
::
get
<
3
>
(
file
);
table_file
.
size
=
std
::
get
<
4
>
(
file
);
table_file
.
date
=
std
::
get
<
5
>
(
file
);
Get
Group
FilePath
(
table_file
);
Get
Table
FilePath
(
table_file
);
if
(
table_file
.
file_type
==
TableFileSchema
::
TO_DELETE
)
{
boost
::
filesystem
::
remove
(
table_file
.
location
);
}
...
...
@@ -675,8 +675,8 @@ Status DBMetaImpl::Count(const std::string& table_id, long& result) {
}
Status
DBMetaImpl
::
DropAll
()
{
if
(
boost
::
filesystem
::
is_directory
(
_options
.
path
))
{
boost
::
filesystem
::
remove_all
(
_options
.
path
);
if
(
boost
::
filesystem
::
is_directory
(
options_
.
path
))
{
boost
::
filesystem
::
remove_all
(
options_
.
path
);
}
return
Status
::
OK
();
}
...
...
cpp/src/db/DBMetaImpl.h
浏览文件 @
2eff64de
...
...
@@ -58,14 +58,14 @@ public:
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
Next
Group
Id
(
std
::
string
&
table_id
);
Status
discard_files_of_size
(
long
to_discard_size
);
std
::
string
Get
Group
Path
(
const
std
::
string
&
table_id
);
std
::
string
Get
Group
DatePartitionPath
(
const
std
::
string
&
table_id
,
DateT
&
date
);
void
Get
Group
FilePath
(
TableFileSchema
&
group_file
);
Status
i
nitialize
();
const
DBMetaOptions
_options
;
Status
Next
Table
Id
(
std
::
string
&
table_id
);
Status
DiscardFiles
(
long
to_discard_size
);
std
::
string
Get
Table
Path
(
const
std
::
string
&
table_id
);
std
::
string
Get
Table
DatePartitionPath
(
const
std
::
string
&
table_id
,
DateT
&
date
);
void
Get
Table
FilePath
(
TableFileSchema
&
group_file
);
Status
I
nitialize
();
const
DBMetaOptions
options_
;
};
// DBMetaImpl
}
// namespace meta
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录