Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
a173486d
M
milvus
项目概览
milvus
/
milvus
9 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
a173486d
编写于
8月 12, 2023
作者:
Y
yah01
提交者:
GitHub
8月 12, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix calculation of memory usage prediction for mmap mode (#26264)
Signed-off-by:
N
yah01
<
yah2er0ne@outlook.com
>
上级
9ca8f280
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
27 addition
and
8 deletion
+27
-8
internal/core/src/mmap/Column.h
internal/core/src/mmap/Column.h
+0
-4
internal/querynodev2/segments/segment_loader.go
internal/querynodev2/segments/segment_loader.go
+27
-4
未找到文件。
internal/core/src/mmap/Column.h
浏览文件 @
a173486d
...
@@ -30,11 +30,7 @@
...
@@ -30,11 +30,7 @@
namespace
milvus
{
namespace
milvus
{
#ifdef MAP_POPULATE
static
int
mmap_flags
=
MAP_SHARED
|
MAP_POPULATE
;
#else
static
int
mmap_flags
=
MAP_SHARED
;
static
int
mmap_flags
=
MAP_SHARED
;
#endif
class
ColumnBase
{
class
ColumnBase
{
public:
public:
...
...
internal/querynodev2/segments/segment_loader.go
浏览文件 @
a173486d
...
@@ -299,6 +299,13 @@ func (loader *segmentLoader) notifyLoadFinish(segments ...*querypb.SegmentLoadIn
...
@@ -299,6 +299,13 @@ func (loader *segmentLoader) notifyLoadFinish(segments ...*querypb.SegmentLoadIn
// requestResource requests memory & storage to load segments,
// requestResource requests memory & storage to load segments,
// returns the memory usage, disk usage and concurrency with the gained memory.
// returns the memory usage, disk usage and concurrency with the gained memory.
func
(
loader
*
segmentLoader
)
requestResource
(
ctx
context
.
Context
,
infos
...*
querypb
.
SegmentLoadInfo
)
(
LoadResource
,
int
,
error
)
{
func
(
loader
*
segmentLoader
)
requestResource
(
ctx
context
.
Context
,
infos
...*
querypb
.
SegmentLoadInfo
)
(
LoadResource
,
int
,
error
)
{
segmentIDs
:=
lo
.
Map
(
infos
,
func
(
info
*
querypb
.
SegmentLoadInfo
,
_
int
)
int64
{
return
info
.
GetSegmentID
()
})
log
:=
log
.
Ctx
(
ctx
)
.
With
(
zap
.
Int64s
(
"segmentIDs"
,
segmentIDs
),
)
resource
:=
LoadResource
{}
resource
:=
LoadResource
{}
loader
.
mut
.
Lock
()
loader
.
mut
.
Lock
()
...
@@ -350,6 +357,14 @@ func (loader *segmentLoader) requestResource(ctx context.Context, infos ...*quer
...
@@ -350,6 +357,14 @@ func (loader *segmentLoader) requestResource(ctx context.Context, infos ...*quer
resource
.
DiskSize
+=
du
resource
.
DiskSize
+=
du
loader
.
committedResource
.
Add
(
resource
)
loader
.
committedResource
.
Add
(
resource
)
log
.
Info
(
"request resource for loading segments"
,
zap
.
Int
(
"workerNum"
,
resource
.
WorkNum
),
zap
.
Int
(
"committedWorkerNum"
,
loader
.
committedResource
.
WorkNum
),
zap
.
Uint64
(
"memory"
,
resource
.
MemorySize
),
zap
.
Uint64
(
"committedMemory"
,
loader
.
committedResource
.
MemorySize
),
zap
.
Uint64
(
"disk"
,
resource
.
DiskSize
),
zap
.
Uint64
(
"committedDisk"
,
loader
.
committedResource
.
DiskSize
),
)
return
resource
,
concurrencyLevel
,
nil
return
resource
,
concurrencyLevel
,
nil
}
}
...
@@ -830,6 +845,7 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
...
@@ -830,6 +845,7 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
metrics
.
QueryNodeDiskUsedSize
.
WithLabelValues
(
fmt
.
Sprint
(
paramtable
.
GetNodeID
()))
.
Set
(
float64
(
toMB
(
uint64
(
localDiskUsage
))))
metrics
.
QueryNodeDiskUsedSize
.
WithLabelValues
(
fmt
.
Sprint
(
paramtable
.
GetNodeID
()))
.
Set
(
float64
(
toMB
(
uint64
(
localDiskUsage
))))
diskUsage
:=
uint64
(
localDiskUsage
)
+
loader
.
committedResource
.
DiskSize
diskUsage
:=
uint64
(
localDiskUsage
)
+
loader
.
committedResource
.
DiskSize
mmapEnabled
:=
len
(
paramtable
.
Get
()
.
QueryNodeCfg
.
MmapDirPath
.
GetValue
())
>
0
maxSegmentSize
:=
uint64
(
0
)
maxSegmentSize
:=
uint64
(
0
)
predictMemUsage
:=
memUsage
predictMemUsage
:=
memUsage
predictDiskUsage
:=
diskUsage
predictDiskUsage
:=
diskUsage
...
@@ -856,10 +872,18 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
...
@@ -856,10 +872,18 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
)
)
return
0
,
0
,
err
return
0
,
0
,
err
}
}
predictMemUsage
+=
neededMemSize
if
mmapEnabled
{
predictDiskUsage
+=
neededDiskSize
predictDiskUsage
+=
neededMemSize
+
neededDiskSize
}
else
{
predictMemUsage
+=
neededMemSize
predictDiskUsage
+=
neededDiskSize
}
}
else
{
}
else
{
predictMemUsage
+=
uint64
(
getBinlogDataSize
(
fieldBinlog
))
if
mmapEnabled
{
predictDiskUsage
+=
uint64
(
getBinlogDataSize
(
fieldBinlog
))
}
else
{
predictMemUsage
+=
uint64
(
getBinlogDataSize
(
fieldBinlog
))
}
}
}
}
}
...
@@ -878,7 +902,6 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
...
@@ -878,7 +902,6 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
}
}
}
}
mmapEnabled
:=
len
(
paramtable
.
Get
()
.
QueryNodeCfg
.
MmapDirPath
.
GetValue
())
>
0
log
.
Info
(
"predict memory and disk usage while loading (in MiB)"
,
log
.
Info
(
"predict memory and disk usage while loading (in MiB)"
,
zap
.
Uint64
(
"maxSegmentSize"
,
toMB
(
maxSegmentSize
)),
zap
.
Uint64
(
"maxSegmentSize"
,
toMB
(
maxSegmentSize
)),
zap
.
Int
(
"concurrency"
,
concurrency
),
zap
.
Int
(
"concurrency"
,
concurrency
),
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录