Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
9618bd9b
M
milvus
项目概览
milvus
/
milvus
10 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
9618bd9b
编写于
7月 26, 2023
作者:
Y
yah01
提交者:
GitHub
7月 26, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Set channel capacity before consuming it (#25895)
Signed-off-by:
N
yah01
<
yang.cen@zilliz.com
>
上级
6f18587f
变更
5
显示空白变更内容
内联
并排
Showing
5 changed file
with
30 addition
and
25 deletion
+30
-25
internal/core/src/index/VectorMemIndex.cpp
internal/core/src/index/VectorMemIndex.cpp
+15
-1
internal/core/src/segcore/SegmentSealedImpl.cpp
internal/core/src/segcore/SegmentSealedImpl.cpp
+4
-0
internal/core/src/segcore/Utils.cpp
internal/core/src/segcore/Utils.cpp
+0
-4
internal/core/src/storage/MemFileManagerImpl.cpp
internal/core/src/storage/MemFileManagerImpl.cpp
+0
-3
internal/core/unittest/test_indexing.cpp
internal/core/unittest/test_indexing.cpp
+11
-17
未找到文件。
internal/core/src/index/VectorMemIndex.cpp
浏览文件 @
9618bd9b
...
...
@@ -33,6 +33,7 @@
#include "common/Consts.h"
#include "common/RangeSearchHelper.h"
#include "common/Utils.h"
#include "log/Log.h"
#include "storage/FieldData.h"
#include "storage/MemFileManagerImpl.h"
#include "storage/ThreadPool.h"
...
...
@@ -103,12 +104,17 @@ VectorMemIndex::Load(const Config& config) {
AssertInfo
(
index_files
.
has_value
(),
"index file paths is empty when load index"
);
auto
parallel_degree
=
static_cast
<
uint64_t
>
(
DEFAULT_FIELD_MAX_MEMORY_LIMIT
/
FILE_SLICE_SIZE
);
std
::
map
<
std
::
string
,
storage
::
FieldDataChannelPtr
>
channels
;
for
(
const
auto
&
file
:
index_files
.
value
())
{
auto
key
=
file
.
substr
(
file
.
find_last_of
(
'/'
)
+
1
);
LOG_SEGCORE_INFO_
<<
"loading index file "
<<
key
;
if
(
channels
.
find
(
key
)
==
channels
.
end
())
{
channels
.
emplace
(
std
::
move
(
key
),
std
::
make_shared
<
storage
::
FieldDataChannel
>
());
std
::
make_shared
<
storage
::
FieldDataChannel
>
(
parallel_degree
*
2
));
}
}
...
...
@@ -116,17 +122,25 @@ VectorMemIndex::Load(const Config& config) {
auto
future
=
pool
.
Submit
(
[
&
]
{
file_manager_
->
LoadFileStream
(
index_files
.
value
(),
channels
);
});
LOG_SEGCORE_INFO_
<<
"assemble index data..."
;
std
::
unordered_map
<
std
::
string
,
storage
::
FieldDataPtr
>
result
;
AssembleIndexDatas
(
channels
,
result
);
LOG_SEGCORE_INFO_
<<
"assemble index data done"
;
LOG_SEGCORE_INFO_
<<
"construct binary set..."
;
BinarySet
binary_set
;
for
(
auto
&
[
key
,
data
]
:
result
)
{
LOG_SEGCORE_INFO_
<<
"add index data to binary set: "
<<
key
;
auto
size
=
data
->
Size
();
auto
deleter
=
[
&
](
uint8_t
*
)
{};
// avoid repeated deconstruction
auto
buf
=
std
::
shared_ptr
<
uint8_t
[]
>
(
(
uint8_t
*
)
const_cast
<
void
*>
(
data
->
Data
()),
deleter
);
binary_set
.
Append
(
key
,
buf
,
size
);
}
LOG_SEGCORE_INFO_
<<
"load index into Knowhere..."
;
LoadWithoutAssemble
(
binary_set
,
config
);
LOG_SEGCORE_INFO_
<<
"load vector index done"
;
}
void
...
...
internal/core/src/segcore/SegmentSealedImpl.cpp
浏览文件 @
9618bd9b
...
...
@@ -182,6 +182,10 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& load_info) {
auto
field_data_info
=
FieldDataInfo
(
field_id
.
get
(),
num_rows
,
load_info
.
mmap_dir_path
);
auto
parallel_degree
=
static_cast
<
uint64_t
>
(
DEFAULT_FIELD_MAX_MEMORY_LIMIT
/
FILE_SLICE_SIZE
);
field_data_info
.
channel
->
set_capacity
(
parallel_degree
*
2
);
auto
&
pool
=
ThreadPool
::
GetInstance
();
auto
load_future
=
pool
.
Submit
(
LoadFieldDatasFromRemote
,
insert_files
,
field_data_info
.
channel
);
...
...
internal/core/src/segcore/Utils.cpp
浏览文件 @
9618bd9b
...
...
@@ -555,10 +555,6 @@ LoadFieldDatasFromRemote(std::vector<std::string>& remote_files,
auto
parallel_degree
=
static_cast
<
uint64_t
>
(
DEFAULT_FIELD_MAX_MEMORY_LIMIT
/
FILE_SLICE_SIZE
);
// set the capacity to 2x parallel_degree, so the memory usage will not be greater than 2x DEFAULT_FIELD_MAX_MEMORY_LIMIT,
// which is 128 MiB
channel
->
set_capacity
(
parallel_degree
*
2
);
auto
rcm
=
storage
::
RemoteChunkManagerSingleton
::
GetInstance
()
.
GetRemoteChunkManager
();
std
::
sort
(
remote_files
.
begin
(),
...
...
internal/core/src/storage/MemFileManagerImpl.cpp
浏览文件 @
9618bd9b
...
...
@@ -125,9 +125,6 @@ MemFileManagerImpl::LoadFileStream(
std
::
map
<
std
::
string
,
storage
::
FieldDataChannelPtr
>&
channels
)
{
auto
parallel_degree
=
static_cast
<
uint64_t
>
(
DEFAULT_FIELD_MAX_MEMORY_LIMIT
/
FILE_SLICE_SIZE
);
for
(
auto
&
[
_
,
channel
]
:
channels
)
{
channel
->
set_capacity
(
parallel_degree
*
2
);
}
std
::
vector
<
std
::
string
>
batch_files
;
auto
LoadBatchIndexFiles
=
[
&
]()
{
...
...
internal/core/unittest/test_indexing.cpp
浏览文件 @
9618bd9b
...
...
@@ -386,9 +386,7 @@ TEST_P(IndexTest, BuildAndQuery) {
milvus
::
index
::
IndexBasePtr
new_index
;
milvus
::
index
::
VectorIndex
*
vec_index
=
nullptr
;
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_DISKANN
)
{
// TODO ::diskann.query need load first, ugly
auto
binary_set
=
index
->
Serialize
(
milvus
::
Config
{});
auto
binary_set
=
index
->
Upload
();
index
.
reset
();
new_index
=
milvus
::
index
::
IndexFactory
::
GetInstance
().
CreateIndex
(
...
...
@@ -400,13 +398,9 @@ TEST_P(IndexTest, BuildAndQuery) {
index_files
.
emplace_back
(
binary
.
first
);
}
load_conf
[
"index_files"
]
=
index_files
;
ASSERT_NO_THROW
(
vec_index
->
Load
(
binary_set
,
load_conf
));
ASSERT_NO_THROW
(
vec_index
->
Load
(
load_conf
));
EXPECT_EQ
(
vec_index
->
Count
(),
NB
);
}
else
{
vec_index
=
dynamic_cast
<
milvus
::
index
::
VectorIndex
*>
(
index
.
get
());
}
EXPECT_EQ
(
vec_index
->
GetDim
(),
DIM
);
EXPECT_EQ
(
vec_index
->
Count
(),
NB
);
milvus
::
SearchInfo
search_info
;
search_info
.
topk_
=
K
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录