Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
5b42a322
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
5b42a322
编写于
10月 26, 2021
作者:
C
cai.zhang
提交者:
GitHub
10月 26, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Increase compatibility for EstimateMemorySize interface (#10603)
Signed-off-by:
N
cai.zhang
<
cai.zhang@zilliz.com
>
上级
0cedd9d4
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
97 addition
and
40 deletion
+97
-40
internal/kv/kv.go
internal/kv/kv.go
+1
-0
internal/kv/mem/mem_kv.go
internal/kv/mem/mem_kv.go
+9
-0
internal/kv/mem/mem_kv_test.go
internal/kv/mem/mem_kv_test.go
+20
-0
internal/kv/minio/minio_kv.go
internal/kv/minio/minio_kv.go
+9
-0
internal/kv/minio/minio_kv_test.go
internal/kv/minio/minio_kv_test.go
+28
-0
internal/querynode/index_loader.go
internal/querynode/index_loader.go
+4
-1
internal/querynode/segment_loader.go
internal/querynode/segment_loader.go
+4
-1
internal/storage/utils.go
internal/storage/utils.go
+1
-37
internal/storage/utils_test.go
internal/storage/utils_test.go
+21
-1
未找到文件。
internal/kv/kv.go
浏览文件 @
5b42a322
...
...
@@ -34,6 +34,7 @@ type BaseKV interface {
type
DataKV
interface
{
BaseKV
LoadPartial
(
key
string
,
start
,
end
int64
)
([]
byte
,
error
)
GetSize
(
key
string
)
(
int64
,
error
)
}
// TxnKV contains extra txn operations of kv. The extra operations is transactional.
...
...
internal/kv/mem/mem_kv.go
浏览文件 @
5b42a322
...
...
@@ -212,3 +212,12 @@ func (kv *MemoryKV) LoadPartial(key string, start, end int64) ([]byte, error) {
start
,
end
)
}
}
func
(
kv
*
MemoryKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
value
,
err
:=
kv
.
Load
(
key
)
if
err
!=
nil
{
return
0
,
err
}
return
int64
(
len
(
value
)),
nil
}
internal/kv/mem/mem_kv_test.go
浏览文件 @
5b42a322
...
...
@@ -56,3 +56,23 @@ func TestMemoryKV_LoadPartial(t *testing.T) {
_
,
err
=
memKV
.
LoadPartial
(
key
,
start
,
end
)
assert
.
Error
(
t
,
err
)
}
func
TestMemoryKV_GetSize
(
t
*
testing
.
T
)
{
memKV
:=
NewMemoryKV
()
key
:=
"TestMemoryKV_GetSize_key"
value
:=
"TestMemoryKV_GetSize_value"
err
:=
memKV
.
Save
(
key
,
value
)
assert
.
NoError
(
t
,
err
)
size
,
err
:=
memKV
.
GetSize
(
key
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
size
,
int64
(
len
(
value
)))
key2
:=
"TestMemoryKV_GetSize_key2"
size
,
err
=
memKV
.
GetSize
(
key2
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
0
),
size
)
}
internal/kv/minio/minio_kv.go
浏览文件 @
5b42a322
...
...
@@ -272,6 +272,15 @@ func (kv *MinIOKV) LoadPartial(key string, start, end int64) ([]byte, error) {
return
ioutil
.
ReadAll
(
object
)
}
func
(
kv
*
MinIOKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
objectInfo
,
err
:=
kv
.
minioClient
.
StatObject
(
kv
.
ctx
,
kv
.
bucketName
,
key
,
minio
.
StatObjectOptions
{})
if
err
!=
nil
{
return
0
,
err
}
return
objectInfo
.
Size
,
nil
}
func
(
kv
*
MinIOKV
)
Close
()
{
}
...
...
internal/kv/minio/minio_kv_test.go
浏览文件 @
5b42a322
...
...
@@ -334,3 +334,31 @@ func TestMinIOKV_FGetObjects(t *testing.T) {
defer
file1
.
Close
()
defer
os
.
Remove
(
path
+
name2
)
}
func
TestMinIOKV_GetSize
(
t
*
testing
.
T
)
{
Params
.
Init
()
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
bucketName
:=
"fantastic-tech-test"
minIOKV
,
err
:=
newMinIOKVClient
(
ctx
,
bucketName
)
assert
.
Nil
(
t
,
err
)
defer
minIOKV
.
RemoveWithPrefix
(
""
)
key
:=
"TestMinIOKV_GetSize_key"
value
:=
"TestMinIOKV_GetSize_value"
err
=
minIOKV
.
Save
(
key
,
value
)
assert
.
NoError
(
t
,
err
)
size
,
err
:=
minIOKV
.
GetSize
(
key
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
size
,
int64
(
len
(
value
)))
key2
:=
"TestMemoryKV_GetSize_key2"
size
,
err
=
minIOKV
.
GetSize
(
key2
)
assert
.
Error
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
0
),
size
)
}
internal/querynode/index_loader.go
浏览文件 @
5b42a322
...
...
@@ -152,7 +152,10 @@ func (loader *indexLoader) estimateIndexBinlogSize(segment *Segment, fieldID Fie
for
_
,
p
:=
range
indexPaths
{
logSize
,
err
:=
storage
.
EstimateMemorySize
(
loader
.
kv
,
p
)
if
err
!=
nil
{
return
0
,
err
logSize
,
err
=
storage
.
GetBinlogSize
(
loader
.
kv
,
p
)
if
err
!=
nil
{
return
0
,
err
}
}
indexSize
+=
logSize
}
...
...
internal/querynode/segment_loader.go
浏览文件 @
5b42a322
...
...
@@ -407,7 +407,10 @@ func (loader *segmentLoader) estimateSegmentSize(segment *Segment,
for
_
,
binlogPath
:=
range
fb
.
Binlogs
{
logSize
,
err
:=
storage
.
EstimateMemorySize
(
loader
.
minioKV
,
binlogPath
)
if
err
!=
nil
{
return
0
,
err
logSize
,
err
=
storage
.
GetBinlogSize
(
loader
.
minioKV
,
binlogPath
)
if
err
!=
nil
{
return
0
,
err
}
}
segmentSize
+=
logSize
}
...
...
internal/storage/utils.go
浏览文件 @
5b42a322
...
...
@@ -26,44 +26,8 @@ import (
// key not in binlog format, size = (a not accurate number), error != nil;
// failed to read event reader, size = (a not accurate number), error != nil;
func
GetBinlogSize
(
kv
kv
.
DataKV
,
key
string
)
(
int64
,
error
)
{
total
:=
int64
(
0
)
header
:=
&
baseEventHeader
{}
headerSize
:=
binary
.
Size
(
header
)
startPos
:=
binary
.
Size
(
MagicNumber
)
endPos
:=
startPos
+
headerSize
for
{
headerContent
,
err
:=
kv
.
LoadPartial
(
key
,
int64
(
startPos
),
int64
(
endPos
))
if
err
!=
nil
{
// case 1: key not exist, total = 0;
// case 2: all events have been read, total = (length of all events);
// whatever the case is, the return value is reasonable.
return
total
,
nil
}
buffer
:=
bytes
.
NewBuffer
(
headerContent
)
header
,
err
:=
readEventHeader
(
buffer
)
if
err
!=
nil
{
// FIXME(dragondriver): should we return 0 here?
return
total
,
fmt
.
Errorf
(
"failed to read event reader: %v"
,
err
)
}
if
header
.
EventLength
<=
0
||
header
.
NextPosition
<
int32
(
endPos
)
{
// key not in binlog format
// FIXME(dragondriver): should we return 0 here?
return
total
,
fmt
.
Errorf
(
"key not in binlog format"
)
}
total
+=
int64
(
header
.
EventLength
)
// startPos = startPos + int(header.EventLength)
// ||
// \/
startPos
=
int
(
header
.
NextPosition
)
endPos
=
startPos
+
headerSize
}
return
kv
.
GetSize
(
key
)
}
// EstimateMemorySize get approximate memory size of a binlog file.
...
...
internal/storage/utils_test.go
浏览文件 @
5b42a322
...
...
@@ -45,6 +45,10 @@ func (kv *mockLessHeaderDataKV) LoadPartial(key string, start, end int64) ([]byt
return
ret
,
nil
}
func
(
kv
*
mockLessHeaderDataKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
return
0
,
errors
.
New
(
"less header"
)
}
func
newMockLessHeaderDataKV
()
*
mockLessHeaderDataKV
{
return
&
mockLessHeaderDataKV
{}
}
...
...
@@ -65,6 +69,10 @@ func (kv *mockWrongHeaderDataKV) LoadPartial(key string, start, end int64) ([]by
return
buffer
.
Bytes
(),
nil
}
func
(
kv
*
mockWrongHeaderDataKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
return
0
,
errors
.
New
(
"wrong header"
)
}
func
newMockWrongHeaderDataKV
()
kv
.
DataKV
{
return
&
mockWrongHeaderDataKV
{}
}
...
...
@@ -121,7 +129,7 @@ func TestGetBinlogSize(t *testing.T) {
size
,
err
=
GetBinlogSize
(
memoryKV
,
blob
.
Key
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
size
+
int64
(
binary
.
Size
(
MagicNumber
))
,
int64
(
len
(
blob
.
Value
)))
assert
.
Equal
(
t
,
size
,
int64
(
len
(
blob
.
Value
)))
}
}
...
...
@@ -243,6 +251,10 @@ func (kv *mockFailedToGetDescDataKV) LoadPartial(key string, start, end int64) (
return
buf
.
Bytes
(),
nil
}
func
(
kv
*
mockFailedToGetDescDataKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
return
0
,
nil
}
func
newMockFailedToGetDescDataKV
()
*
mockFailedToGetDescDataKV
{
return
&
mockFailedToGetDescDataKV
{}
}
...
...
@@ -282,6 +294,10 @@ func (kv *mockLessDescDataKV) LoadPartial(key string, start, end int64) ([]byte,
*/
}
func
(
kv
*
mockLessDescDataKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
return
0
,
nil
}
func
newMockLessDescDataKV
()
*
mockLessDescDataKV
{
return
&
mockLessDescDataKV
{}
}
...
...
@@ -307,6 +323,10 @@ func (kv *mockOriginalSizeDataKV) LoadPartial(key string, start, end int64) ([]b
return
nil
,
nil
}
func
(
kv
*
mockOriginalSizeDataKV
)
GetSize
(
key
string
)
(
int64
,
error
)
{
return
0
,
nil
}
func
newMockOriginalSizeDataKV
()
*
mockOriginalSizeDataKV
{
return
&
mockOriginalSizeDataKV
{}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录