Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
c80d8f82
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
c80d8f82
编写于
5月 06, 2023
作者:
C
cai.zhang
提交者:
GitHub
5月 06, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Reduce memory usage for diskann index (#23600)
Signed-off-by:
N
cai.zhang
<
cai.zhang@zilliz.com
>
上级
6a48de34
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
67 addition
and
25 deletion
+67
-25
internal/core/src/storage/DiskFileManagerImpl.cpp
internal/core/src/storage/DiskFileManagerImpl.cpp
+54
-24
internal/core/src/storage/DiskFileManagerImpl.h
internal/core/src/storage/DiskFileManagerImpl.h
+13
-1
未找到文件。
internal/core/src/storage/DiskFileManagerImpl.cpp
浏览文件 @
c80d8f82
...
...
@@ -123,42 +123,72 @@ DiskFileManagerImpl::AddFile(const std::string& file) noexcept {
auto
fileName
=
GetFileName
(
file
);
auto
fileSize
=
local_chunk_manager
.
Size
(
file
);
// Split local data to multi part with specified size
std
::
vector
<
std
::
string
>
batch_remote_files
;
std
::
vector
<
int64_t
>
remote_file_sizes
;
std
::
vector
<
int64_t
>
local_file_offsets
;
int
slice_num
=
0
;
auto
remotePrefix
=
GetRemoteIndexObjectPrefix
();
std
::
vector
<
std
::
future
<
std
::
pair
<
std
::
string
,
size_t
>>>
futures
;
auto
parallel_degree
=
uint64_t
(
DEFAULT_DISK_INDEX_MAX_MEMORY_LIMIT
/
(
index_file_slice_size
<<
20
))
;
for
(
int64_t
offset
=
0
;
offset
<
fileSize
;
slice_num
++
)
{
if
(
batch_remote_files
.
size
()
>=
parallel_degree
)
{
AddBatchIndexFiles
(
file
,
local_file_offsets
,
batch_remote_files
,
remote_file_sizes
);
batch_remote_files
.
clear
();
remote_file_sizes
.
clear
();
local_file_offsets
.
clear
();
}
auto
batch_size
=
std
::
min
(
index_file_slice_size
<<
20
,
int64_t
(
fileSize
)
-
offset
);
// Put file to remote
char
objectKey
[
200
];
snprintf
(
objectKey
,
sizeof
(
objectKey
),
"%s/%s_%d"
,
remotePrefix
.
c_str
(),
fileName
.
c_str
(),
slice_num
);
// use multi-thread to put part file
batch_remote_files
.
emplace_back
(
GenerateRemoteIndexFile
(
fileName
,
slice_num
));
remote_file_sizes
.
emplace_back
(
batch_size
);
local_file_offsets
.
emplace_back
(
offset
);
offset
+=
batch_size
;
}
if
(
batch_remote_files
.
size
()
>
0
)
{
AddBatchIndexFiles
(
file
,
local_file_offsets
,
batch_remote_files
,
remote_file_sizes
);
}
FILEMANAGER_CATCH
FILEMANAGER_END
return
true
;
}
// namespace knowhere
void
DiskFileManagerImpl
::
AddBatchIndexFiles
(
const
std
::
string
&
local_file_name
,
const
std
::
vector
<
int64_t
>&
local_file_offsets
,
const
std
::
vector
<
std
::
string
>&
remote_files
,
const
std
::
vector
<
int64_t
>&
remote_file_sizes
)
{
auto
&
pool
=
ThreadPool
::
GetInstance
();
std
::
vector
<
std
::
future
<
std
::
pair
<
std
::
string
,
size_t
>>>
futures
;
AssertInfo
(
local_file_offsets
.
size
()
==
remote_files
.
size
(),
"inconsistent size of offset slices with file slices"
);
AssertInfo
(
remote_files
.
size
()
==
remote_file_sizes
.
size
(),
"inconsistent size of file slices with size slices"
);
for
(
int64_t
i
=
0
;
i
<
remote_files
.
size
();
++
i
)
{
futures
.
push_back
(
pool
.
Submit
(
EncodeAndUploadIndexSlice
,
rcm_
.
get
(),
fil
e
,
offset
,
batch_size
,
local_file_nam
e
,
local_file_offsets
[
i
]
,
remote_file_sizes
[
i
]
,
index_meta_
,
field_meta_
,
std
::
string
(
objectKey
)));
offset
+=
batch_size
;
remote_files
[
i
]));
}
for
(
auto
&
future
:
futures
)
{
auto
res
=
future
.
get
();
remote_paths_to_size_
[
res
.
first
]
=
res
.
second
;
}
FILEMANAGER_CATCH
FILEMANAGER_END
return
true
;
}
// namespace knowhere
}
void
DiskFileManagerImpl
::
CacheIndexToDisk
(
std
::
vector
<
std
::
string
>
remote_files
)
{
...
...
@@ -262,7 +292,7 @@ DiskFileManagerImpl::GetFileName(const std::string& localfile) {
}
std
::
string
DiskFileManagerImpl
::
GetRemoteIndexObjectPrefix
()
{
DiskFileManagerImpl
::
GetRemoteIndexObjectPrefix
()
const
{
return
remote_root_path_
+
"/"
+
std
::
string
(
INDEX_ROOT_PATH
)
+
"/"
+
std
::
to_string
(
index_meta_
.
build_id
)
+
"/"
+
std
::
to_string
(
index_meta_
.
index_version
)
+
"/"
+
...
...
internal/core/src/storage/DiskFileManagerImpl.h
浏览文件 @
c80d8f82
...
...
@@ -58,7 +58,7 @@ class DiskFileManagerImpl : public FileManagerImpl {
}
std
::
string
GetRemoteIndexObjectPrefix
();
GetRemoteIndexObjectPrefix
()
const
;
std
::
string
GetLocalIndexObjectPrefix
();
...
...
@@ -76,6 +76,12 @@ class DiskFileManagerImpl : public FileManagerImpl {
return
local_paths_
;
}
std
::
string
GenerateRemoteIndexFile
(
std
::
string
file_name
,
int64_t
slice_num
)
const
{
return
GetRemoteIndexObjectPrefix
()
+
"/"
+
file_name
+
"_"
+
std
::
to_string
(
slice_num
);
}
void
CacheIndexToDisk
(
std
::
vector
<
std
::
string
>
remote_files
);
...
...
@@ -84,6 +90,12 @@ class DiskFileManagerImpl : public FileManagerImpl {
const
std
::
string
&
local_file_name
,
uint64_t
local_file_init_offfset
);
void
AddBatchIndexFiles
(
const
std
::
string
&
local_file_name
,
const
std
::
vector
<
int64_t
>&
local_file_offsets
,
const
std
::
vector
<
std
::
string
>&
remote_files
,
const
std
::
vector
<
int64_t
>&
remote_file_sizes
);
FieldDataMeta
GetFileDataMeta
()
const
{
return
field_meta_
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录