Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
9f38b96e
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9f38b96e
编写于
7月 05, 2019
作者:
Z
zhiru
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Implemented add and serialize
Former-commit-id: 25fbbc2185efc4b45ea8f4693fea0ba0001d267e
上级
8f42ef67
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
89 addition
and
28 deletion
+89
-28
cpp/src/db/MemTable.cpp
cpp/src/db/MemTable.cpp
+23
-9
cpp/src/db/MemTable.h
cpp/src/db/MemTable.h
+7
-3
cpp/src/db/MemTableFile.cpp
cpp/src/db/MemTableFile.cpp
+39
-3
cpp/src/db/MemTableFile.h
cpp/src/db/MemTableFile.h
+6
-2
cpp/src/db/VectorSource.cpp
cpp/src/db/VectorSource.cpp
+6
-6
cpp/src/db/VectorSource.h
cpp/src/db/VectorSource.h
+0
-2
cpp/unittest/db/mem_test.cpp
cpp/unittest/db/mem_test.cpp
+8
-3
未找到文件。
cpp/src/db/MemTable.cpp
浏览文件 @
9f38b96e
...
...
@@ -6,24 +6,26 @@ namespace milvus {
namespace
engine
{
MemTable
::
MemTable
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
)
:
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
,
const
Options
&
options
)
:
table_id_
(
table_id
),
meta_
(
meta
)
{
meta_
(
meta
),
options_
(
options
)
{
}
Status
MemTable
::
Add
(
VectorSource
::
Ptr
&
source
)
{
while
(
!
source
->
AllAdded
())
{
MemTableFile
::
Ptr
currentMemTableFile
;
if
(
!
mem_table_file_
stack
_
.
empty
())
{
currentMemTableFile
=
mem_table_file_
stack_
.
top
();
if
(
!
mem_table_file_
list
_
.
empty
())
{
currentMemTableFile
=
mem_table_file_
list_
.
back
();
}
Status
status
;
if
(
mem_table_file_
stack_
.
empty
()
||
currentMemTableFile
->
i
sFull
())
{
MemTableFile
::
Ptr
newMemTableFile
=
std
::
make_shared
<
MemTableFile
>
(
table_id_
,
meta_
);
if
(
mem_table_file_
list_
.
empty
()
||
currentMemTableFile
->
I
sFull
())
{
MemTableFile
::
Ptr
newMemTableFile
=
std
::
make_shared
<
MemTableFile
>
(
table_id_
,
meta_
,
options_
);
status
=
newMemTableFile
->
Add
(
source
);
if
(
status
.
ok
())
{
mem_table_file_
stack_
.
push
(
newMemTableFile
);
mem_table_file_
list_
.
emplace_back
(
newMemTableFile
);
}
}
else
{
...
...
@@ -39,11 +41,23 @@ Status MemTable::Add(VectorSource::Ptr& source) {
}
void
MemTable
::
GetCurrentMemTableFile
(
MemTableFile
::
Ptr
&
mem_table_file
)
{
mem_table_file
=
mem_table_file_
stack_
.
top
();
mem_table_file
=
mem_table_file_
list_
.
back
();
}
size_t
MemTable
::
GetStackSize
()
{
return
mem_table_file_stack_
.
size
();
return
mem_table_file_list_
.
size
();
}
Status
MemTable
::
Serialize
()
{
for
(
auto
&
memTableFile
:
mem_table_file_list_
)
{
auto
status
=
memTableFile
->
Serialize
();
if
(
!
status
.
ok
())
{
std
::
string
errMsg
=
"MemTable::Serialize failed: "
+
status
.
ToString
();
ENGINE_LOG_ERROR
<<
errMsg
;
return
Status
::
Error
(
errMsg
);
}
}
return
Status
::
OK
();
}
}
// namespace engine
...
...
cpp/src/db/MemTable.h
浏览文件 @
9f38b96e
...
...
@@ -15,10 +15,10 @@ class MemTable {
public:
using
Ptr
=
std
::
shared_ptr
<
MemTable
>
;
using
MemTableFile
Stack
=
std
::
stack
<
MemTableFile
::
Ptr
>
;
using
MemTableFile
List
=
std
::
vector
<
MemTableFile
::
Ptr
>
;
using
MetaPtr
=
meta
::
Meta
::
Ptr
;
MemTable
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
);
MemTable
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
,
const
Options
&
options
);
Status
Add
(
VectorSource
::
Ptr
&
source
);
...
...
@@ -26,13 +26,17 @@ public:
size_t
GetStackSize
();
Status
Serialize
();
private:
const
std
::
string
table_id_
;
MemTableFile
Stack
mem_table_file_stack
_
;
MemTableFile
List
mem_table_file_list
_
;
MetaPtr
meta_
;
Options
options_
;
};
//MemTable
}
// namespace engine
...
...
cpp/src/db/MemTableFile.cpp
浏览文件 @
9f38b96e
...
...
@@ -2,6 +2,7 @@
#include "Constants.h"
#include "Log.h"
#include "EngineFactory.h"
#include "metrics/Metrics.h"
#include <cmath>
...
...
@@ -10,9 +11,11 @@ namespace milvus {
namespace
engine
{
MemTableFile
::
MemTableFile
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
)
:
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
,
const
Options
&
options
)
:
table_id_
(
table_id
),
meta_
(
meta
)
{
meta_
(
meta
),
options_
(
options
)
{
current_mem_
=
0
;
auto
status
=
CreateTableFile
();
...
...
@@ -40,6 +43,13 @@ Status MemTableFile::CreateTableFile() {
Status
MemTableFile
::
Add
(
const
VectorSource
::
Ptr
&
source
)
{
if
(
table_file_schema_
.
dimension_
<=
0
)
{
std
::
string
errMsg
=
"MemTableFile::Add: table_file_schema dimension = "
+
std
::
to_string
(
table_file_schema_
.
dimension_
)
+
", table_id = "
+
table_file_schema_
.
table_id_
;
ENGINE_LOG_ERROR
<<
errMsg
;
return
Status
::
Error
(
errMsg
);
}
size_t
singleVectorMemSize
=
table_file_schema_
.
dimension_
*
VECTOR_TYPE_SIZE
;
size_t
memLeft
=
GetMemLeft
();
if
(
memLeft
>=
singleVectorMemSize
)
{
...
...
@@ -62,11 +72,37 @@ size_t MemTableFile::GetMemLeft() {
return
(
MAX_TABLE_FILE_MEM
-
current_mem_
);
}
bool
MemTableFile
::
i
sFull
()
{
bool
MemTableFile
::
I
sFull
()
{
size_t
singleVectorMemSize
=
table_file_schema_
.
dimension_
*
VECTOR_TYPE_SIZE
;
return
(
GetMemLeft
()
<
singleVectorMemSize
);
}
Status
MemTableFile
::
Serialize
()
{
auto
start_time
=
METRICS_NOW_TIME
;
auto
size
=
GetCurrentMem
();
execution_engine_
->
Serialize
();
auto
end_time
=
METRICS_NOW_TIME
;
auto
total_time
=
METRICS_MICROSECONDS
(
start_time
,
end_time
);
table_file_schema_
.
size_
=
size
;
server
::
Metrics
::
GetInstance
().
DiskStoreIOSpeedGaugeSet
((
double
)
size
/
total_time
);
table_file_schema_
.
file_type_
=
(
size
>=
options_
.
index_trigger_size
)
?
meta
::
TableFileSchema
::
TO_INDEX
:
meta
::
TableFileSchema
::
RAW
;
auto
status
=
meta_
->
UpdateTableFile
(
table_file_schema_
);
LOG
(
DEBUG
)
<<
"New "
<<
((
table_file_schema_
.
file_type_
==
meta
::
TableFileSchema
::
RAW
)
?
"raw"
:
"to_index"
)
<<
" file "
<<
table_file_schema_
.
file_id_
<<
" of size "
<<
(
double
)
size
/
(
double
)
M
<<
" M"
;
execution_engine_
->
Cache
();
return
status
;
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
cpp/src/db/MemTableFile.h
浏览文件 @
9f38b96e
...
...
@@ -16,7 +16,7 @@ public:
using
Ptr
=
std
::
shared_ptr
<
MemTableFile
>
;
using
MetaPtr
=
meta
::
Meta
::
Ptr
;
MemTableFile
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
);
MemTableFile
(
const
std
::
string
&
table_id
,
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
,
const
Options
&
options
);
Status
Add
(
const
VectorSource
::
Ptr
&
source
);
...
...
@@ -24,7 +24,9 @@ public:
size_t
GetMemLeft
();
bool
isFull
();
bool
IsFull
();
Status
Serialize
();
private:
...
...
@@ -36,6 +38,8 @@ private:
MetaPtr
meta_
;
Options
options_
;
size_t
current_mem_
;
ExecutionEnginePtr
execution_engine_
;
...
...
cpp/src/db/VectorSource.cpp
浏览文件 @
9f38b96e
...
...
@@ -2,6 +2,7 @@
#include "ExecutionEngine.h"
#include "EngineFactory.h"
#include "Log.h"
#include "metrics/Metrics.h"
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -21,12 +22,7 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine,
const
size_t
&
num_vectors_to_add
,
size_t
&
num_vectors_added
)
{
if
(
table_file_schema
.
dimension_
<=
0
)
{
std
::
string
errMsg
=
"VectorSource::Add: table_file_schema dimension = "
+
std
::
to_string
(
table_file_schema
.
dimension_
)
+
", table_id = "
+
table_file_schema
.
table_id_
;
ENGINE_LOG_ERROR
<<
errMsg
;
return
Status
::
Error
(
errMsg
);
}
auto
start_time
=
METRICS_NOW_TIME
;
num_vectors_added
=
current_num_vectors_added
+
num_vectors_to_add
<=
n_
?
num_vectors_to_add
:
n_
-
current_num_vectors_added
;
IDNumbers
vector_ids_to_add
;
...
...
@@ -40,6 +36,10 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine,
ENGINE_LOG_ERROR
<<
"VectorSource::Add failed: "
+
status
.
ToString
();
}
auto
end_time
=
METRICS_NOW_TIME
;
auto
total_time
=
METRICS_MICROSECONDS
(
start_time
,
end_time
);
server
::
Metrics
::
GetInstance
().
AddVectorsPerSecondGaugeSet
(
static_cast
<
int
>
(
n_
),
static_cast
<
int
>
(
table_file_schema
.
dimension_
),
total_time
);
return
status
;
}
...
...
cpp/src/db/VectorSource.h
浏览文件 @
9f38b96e
...
...
@@ -28,8 +28,6 @@ public:
IDNumbers
GetVectorIds
();
// Status Serialize();
private:
const
size_t
n_
;
...
...
cpp/unittest/db/mem_test.cpp
浏览文件 @
9f38b96e
...
...
@@ -86,12 +86,13 @@ TEST(MEM_TEST, VECTOR_SOURCE_TEST) {
TEST
(
MEM_TEST
,
MEM_TABLE_FILE_TEST
)
{
std
::
shared_ptr
<
engine
::
meta
::
DBMetaImpl
>
impl_
=
engine
::
DBMetaImplFactory
::
Build
();
auto
options
=
engine
::
OptionsFactory
::
Build
();
engine
::
meta
::
TableSchema
table_schema
=
BuildTableSchema
();
auto
status
=
impl_
->
CreateTable
(
table_schema
);
ASSERT_TRUE
(
status
.
ok
());
engine
::
MemTableFile
memTableFile
(
TABLE_NAME
,
impl_
);
engine
::
MemTableFile
memTableFile
(
TABLE_NAME
,
impl_
,
options
);
int64_t
n_100
=
100
;
std
::
vector
<
float
>
vectors_100
;
...
...
@@ -120,7 +121,7 @@ TEST(MEM_TEST, MEM_TABLE_FILE_TEST) {
vector_ids
=
source_128M
->
GetVectorIds
();
ASSERT_EQ
(
vector_ids
.
size
(),
n_max
-
n_100
);
ASSERT_TRUE
(
memTableFile
.
i
sFull
());
ASSERT_TRUE
(
memTableFile
.
I
sFull
());
status
=
impl_
->
DropAll
();
ASSERT_TRUE
(
status
.
ok
());
...
...
@@ -129,6 +130,7 @@ TEST(MEM_TEST, MEM_TABLE_FILE_TEST) {
TEST
(
MEM_TEST
,
MEM_TABLE_TEST
)
{
std
::
shared_ptr
<
engine
::
meta
::
DBMetaImpl
>
impl_
=
engine
::
DBMetaImplFactory
::
Build
();
auto
options
=
engine
::
OptionsFactory
::
Build
();
engine
::
meta
::
TableSchema
table_schema
=
BuildTableSchema
();
auto
status
=
impl_
->
CreateTable
(
table_schema
);
...
...
@@ -140,7 +142,7 @@ TEST(MEM_TEST, MEM_TABLE_TEST) {
engine
::
VectorSource
::
Ptr
source_100
=
std
::
make_shared
<
engine
::
VectorSource
>
(
n_100
,
vectors_100
.
data
());
engine
::
MemTable
memTable
(
TABLE_NAME
,
impl_
);
engine
::
MemTable
memTable
(
TABLE_NAME
,
impl_
,
options
);
status
=
memTable
.
Add
(
source_100
);
ASSERT_TRUE
(
status
.
ok
());
...
...
@@ -184,6 +186,9 @@ TEST(MEM_TEST, MEM_TABLE_TEST) {
int
expectedStackSize
=
2
+
std
::
ceil
((
n_1G
-
n_100
)
*
singleVectorMem
/
engine
::
MAX_TABLE_FILE_MEM
);
ASSERT_EQ
(
memTable
.
GetStackSize
(),
expectedStackSize
);
status
=
memTable
.
Serialize
();
ASSERT_TRUE
(
status
.
ok
());
status
=
impl_
->
DropAll
();
ASSERT_TRUE
(
status
.
ok
());
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录