Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
2be217bc
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
2be217bc
编写于
3月 03, 2022
作者:
L
Letian Jiang
提交者:
GitHub
3月 03, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Specify appropriate timestamp in initPKBloomFilter (#15823) (#15826)
Signed-off-by:
N
Letian Jiang
<
letian.jiang@zilliz.com
>
上级
3f34bc1a
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
24 addition
and
23 deletion
+24
-23
internal/datanode/data_sync_service.go
internal/datanode/data_sync_service.go
+4
-3
internal/datanode/segment_replica.go
internal/datanode/segment_replica.go
+8
-8
internal/datanode/segment_replica_test.go
internal/datanode/segment_replica_test.go
+12
-12
未找到文件。
internal/datanode/data_sync_service.go
浏览文件 @
2be217bc
...
...
@@ -170,7 +170,8 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo) erro
pos
:
*
us
.
GetDmlPosition
(),
}
}
if
err
:=
dsService
.
replica
.
addNormalSegment
(
us
.
GetID
(),
us
.
CollectionID
,
us
.
PartitionID
,
us
.
GetInsertChannel
(),
us
.
GetNumOfRows
(),
us
.
Statslogs
,
cp
);
err
!=
nil
{
if
err
:=
dsService
.
replica
.
addNormalSegment
(
us
.
GetID
(),
us
.
CollectionID
,
us
.
PartitionID
,
us
.
GetInsertChannel
(),
us
.
GetNumOfRows
(),
us
.
Statslogs
,
cp
,
vchanInfo
.
GetSeekPosition
()
.
GetTimestamp
());
err
!=
nil
{
return
err
}
}
...
...
@@ -192,8 +193,8 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo) erro
zap
.
Int64
(
"SegmentID"
,
fs
.
GetID
()),
zap
.
Int64
(
"NumOfRows"
,
fs
.
GetNumOfRows
()),
)
if
err
:=
dsService
.
replica
.
addFlushedSegment
(
fs
.
GetID
(),
fs
.
CollectionID
,
fs
.
PartitionID
,
fs
.
GetInsertChannel
(),
fs
.
GetNumOfRows
(),
fs
.
Statslogs
);
err
!=
nil
{
if
err
:=
dsService
.
replica
.
addFlushedSegment
(
fs
.
GetID
(),
fs
.
CollectionID
,
fs
.
PartitionID
,
fs
.
GetInsertChannel
(),
fs
.
GetNumOfRows
(),
fs
.
Statslogs
,
vchanInfo
.
GetSeekPosition
()
.
GetTimestamp
()
);
err
!=
nil
{
return
err
}
}
...
...
internal/datanode/segment_replica.go
浏览文件 @
2be217bc
...
...
@@ -52,9 +52,9 @@ type Replica interface {
listAllSegmentIDs
()
[]
UniqueID
addNewSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
startPos
,
endPos
*
internalpb
.
MsgPosition
)
error
addNormalSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlog
[]
*
datapb
.
FieldBinlog
,
cp
*
segmentCheckPoint
)
error
addNormalSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlog
[]
*
datapb
.
FieldBinlog
,
cp
*
segmentCheckPoint
,
recoverTs
Timestamp
)
error
filterSegments
(
channelName
string
,
partitionID
UniqueID
)
[]
*
Segment
addFlushedSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlog
[]
*
datapb
.
FieldBinlog
)
error
addFlushedSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlog
[]
*
datapb
.
FieldBinlog
,
recoverTs
Timestamp
)
error
listNewSegmentsStartPositions
()
[]
*
datapb
.
SegmentStartPosition
listSegmentsCheckPoints
()
map
[
UniqueID
]
segmentCheckPoint
updateSegmentEndPosition
(
segID
UniqueID
,
endPos
*
internalpb
.
MsgPosition
)
...
...
@@ -322,7 +322,7 @@ func (replica *SegmentReplica) filterSegments(channelName string, partitionID Un
// addNormalSegment adds a *NotNew* and *NotFlushed* segment. Before add, please make sure there's no
// such segment by `hasSegment`
func
(
replica
*
SegmentReplica
)
addNormalSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
,
cp
*
segmentCheckPoint
)
error
{
func
(
replica
*
SegmentReplica
)
addNormalSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
,
cp
*
segmentCheckPoint
,
recoverTs
Timestamp
)
error
{
if
collID
!=
replica
.
collectionID
{
log
.
Warn
(
"Mismatch collection"
,
zap
.
Int64
(
"input ID"
,
collID
),
...
...
@@ -352,7 +352,7 @@ func (replica *SegmentReplica) addNormalSegment(segID, collID, partitionID Uniqu
seg
.
checkPoint
=
*
cp
seg
.
endPos
=
&
cp
.
pos
}
err
:=
replica
.
initPKBloomFilter
(
seg
,
statsBinlogs
)
err
:=
replica
.
initPKBloomFilter
(
seg
,
statsBinlogs
,
recoverTs
)
if
err
!=
nil
{
return
err
}
...
...
@@ -370,7 +370,7 @@ func (replica *SegmentReplica) addNormalSegment(segID, collID, partitionID Uniqu
// addFlushedSegment adds a *Flushed* segment. Before add, please make sure there's no
// such segment by `hasSegment`
func
(
replica
*
SegmentReplica
)
addFlushedSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
)
error
{
func
(
replica
*
SegmentReplica
)
addFlushedSegment
(
segID
,
collID
,
partitionID
UniqueID
,
channelName
string
,
numOfRows
int64
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
,
recoverTs
Timestamp
)
error
{
if
collID
!=
replica
.
collectionID
{
log
.
Warn
(
"Mismatch collection"
,
...
...
@@ -399,7 +399,7 @@ func (replica *SegmentReplica) addFlushedSegment(segID, collID, partitionID Uniq
maxPK
:
math
.
MinInt64
,
// use min value represents no value
}
err
:=
replica
.
initPKBloomFilter
(
seg
,
statsBinlogs
)
err
:=
replica
.
initPKBloomFilter
(
seg
,
statsBinlogs
,
recoverTs
)
if
err
!=
nil
{
return
err
}
...
...
@@ -414,9 +414,9 @@ func (replica *SegmentReplica) addFlushedSegment(segID, collID, partitionID Uniq
return
nil
}
func
(
replica
*
SegmentReplica
)
initPKBloomFilter
(
s
*
Segment
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
)
error
{
func
(
replica
*
SegmentReplica
)
initPKBloomFilter
(
s
*
Segment
,
statsBinlogs
[]
*
datapb
.
FieldBinlog
,
ts
Timestamp
)
error
{
log
.
Info
(
"begin to init pk bloom filter"
,
zap
.
Int
(
"stats bin logs"
,
len
(
statsBinlogs
)))
schema
,
err
:=
replica
.
getCollectionSchema
(
s
.
collectionID
,
0
)
schema
,
err
:=
replica
.
getCollectionSchema
(
s
.
collectionID
,
ts
)
if
err
!=
nil
{
return
err
}
...
...
internal/datanode/segment_replica_test.go
浏览文件 @
2be217bc
...
...
@@ -340,7 +340,7 @@ func TestSegmentReplica_InterfaceMethod(t *testing.T) {
sr
.
minIOKV
=
&
mockMinioKV
{}
assert
.
Nil
(
t
,
err
)
require
.
False
(
t
,
sr
.
hasSegment
(
test
.
inSegID
,
true
))
err
=
sr
.
addNormalSegment
(
test
.
inSegID
,
test
.
inCollID
,
1
,
""
,
0
,
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
&
segmentCheckPoint
{})
err
=
sr
.
addNormalSegment
(
test
.
inSegID
,
test
.
inCollID
,
1
,
""
,
0
,
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
&
segmentCheckPoint
{}
,
0
)
if
test
.
isValidCase
{
assert
.
NoError
(
t
,
err
)
assert
.
True
(
t
,
sr
.
hasSegment
(
test
.
inSegID
,
true
))
...
...
@@ -361,7 +361,7 @@ func TestSegmentReplica_InterfaceMethod(t *testing.T) {
segID
:=
int64
(
101
)
require
.
False
(
t
,
sr
.
hasSegment
(
segID
,
true
))
assert
.
NotPanics
(
t
,
func
()
{
err
=
sr
.
addNormalSegment
(
segID
,
1
,
10
,
"empty_dml_chan"
,
0
,
[]
*
datapb
.
FieldBinlog
{},
nil
)
err
=
sr
.
addNormalSegment
(
segID
,
1
,
10
,
"empty_dml_chan"
,
0
,
[]
*
datapb
.
FieldBinlog
{},
nil
,
0
)
assert
.
NoError
(
t
,
err
)
})
})
...
...
@@ -587,9 +587,9 @@ func TestSegmentReplica_InterfaceMethod(t *testing.T) {
cpPos
:=
&
internalpb
.
MsgPosition
{
ChannelName
:
"insert-01"
,
Timestamp
:
Timestamp
(
10
)}
cp
:=
&
segmentCheckPoint
{
int64
(
10
),
*
cpPos
}
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
)
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
,
0
)
assert
.
NotNil
(
t
,
err
)
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()})
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()}
,
0
)
assert
.
NotNil
(
t
,
err
)
})
...
...
@@ -600,9 +600,9 @@ func TestSegmentReplica_InterfaceMethod(t *testing.T) {
cpPos
:=
&
internalpb
.
MsgPosition
{
ChannelName
:
"insert-01"
,
Timestamp
:
Timestamp
(
10
)}
cp
:=
&
segmentCheckPoint
{
int64
(
10
),
*
cpPos
}
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
)
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
,
0
)
assert
.
NotNil
(
t
,
err
)
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()})
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()}
,
0
)
assert
.
NotNil
(
t
,
err
)
})
...
...
@@ -613,9 +613,9 @@ func TestSegmentReplica_InterfaceMethod(t *testing.T) {
cpPos
:=
&
internalpb
.
MsgPosition
{
ChannelName
:
"insert-01"
,
Timestamp
:
Timestamp
(
10
)}
cp
:=
&
segmentCheckPoint
{
int64
(
10
),
*
cpPos
}
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
)
err
=
sr
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
,
0
)
assert
.
NotNil
(
t
,
err
)
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()})
err
=
sr
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()}
,
0
)
assert
.
NotNil
(
t
,
err
)
})
...
...
@@ -678,7 +678,7 @@ func TestInnerFunctionSegment(t *testing.T) {
cpPos
:=
&
internalpb
.
MsgPosition
{
ChannelName
:
"insert-01"
,
Timestamp
:
Timestamp
(
10
)}
cp
:=
&
segmentCheckPoint
{
int64
(
10
),
*
cpPos
}
err
=
replica
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
)
err
=
replica
.
addNormalSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
10
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
,
0
)
assert
.
NoError
(
t
,
err
)
assert
.
True
(
t
,
replica
.
hasSegment
(
1
,
true
))
assert
.
Equal
(
t
,
1
,
len
(
replica
.
normalSegments
))
...
...
@@ -695,7 +695,7 @@ func TestInnerFunctionSegment(t *testing.T) {
assert
.
False
(
t
,
seg
.
isNew
.
Load
()
.
(
bool
))
assert
.
False
(
t
,
seg
.
isFlushed
.
Load
()
.
(
bool
))
err
=
replica
.
addNormalSegment
(
1
,
100000
,
2
,
"invalid"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
&
segmentCheckPoint
{})
err
=
replica
.
addNormalSegment
(
1
,
100000
,
2
,
"invalid"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
&
segmentCheckPoint
{}
,
0
)
assert
.
Error
(
t
,
err
)
replica
.
updateStatistics
(
1
,
10
)
...
...
@@ -730,7 +730,7 @@ func TestInnerFunctionSegment(t *testing.T) {
replica
.
updateSegmentCheckPoint
(
1
)
assert
.
Equal
(
t
,
int64
(
20
),
replica
.
normalSegments
[
UniqueID
(
1
)]
.
checkPoint
.
numRows
)
err
=
replica
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()})
err
=
replica
.
addFlushedSegment
(
1
,
1
,
2
,
"insert-01"
,
int64
(
0
),
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()}
,
0
)
assert
.
Nil
(
t
,
err
)
totalSegments
:=
replica
.
filterSegments
(
"insert-01"
,
common
.
InvalidPartitionID
)
...
...
@@ -776,7 +776,7 @@ func TestReplica_UpdatePKRange(t *testing.T) {
err
=
replica
.
addNewSegment
(
1
,
collID
,
partID
,
chanName
,
startPos
,
endPos
)
assert
.
Nil
(
t
,
err
)
err
=
replica
.
addNormalSegment
(
2
,
collID
,
partID
,
chanName
,
100
,
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
)
err
=
replica
.
addNormalSegment
(
2
,
collID
,
partID
,
chanName
,
100
,
[]
*
datapb
.
FieldBinlog
{
getSimpleFieldBinlog
()},
cp
,
0
)
assert
.
Nil
(
t
,
err
)
segNew
:=
replica
.
newSegments
[
1
]
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录