Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
c9174d55
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
c9174d55
编写于
7月 28, 2022
作者:
J
jaime
提交者:
GitHub
7月 28, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refine merge operation during compacting phase (#18399)
Signed-off-by:
N
yun.zhang
<
yun.zhang@zilliz.com
>
上级
a287a2b3
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
27 addition
and
18 deletion
+27
-18
internal/datacoord/compaction_trigger.go
internal/datacoord/compaction_trigger.go
+6
-1
internal/datanode/compactor.go
internal/datanode/compactor.go
+7
-9
internal/datanode/compactor_test.go
internal/datanode/compactor_test.go
+5
-8
internal/storage/primary_key.go
internal/storage/primary_key.go
+9
-0
未找到文件。
internal/datacoord/compaction_trigger.go
浏览文件 @
c9174d55
...
...
@@ -253,7 +253,12 @@ func (t *compactionTrigger) handleGlobalSignal(signal *compactionSignal) {
continue
}
log
.
Info
(
"time cost of generating global compaction"
,
zap
.
Int64
(
"planID"
,
plan
.
PlanID
),
zap
.
Any
(
"time cost"
,
time
.
Since
(
start
)
.
Milliseconds
()),
segIDs
:=
make
(
map
[
int64
][]
*
datapb
.
FieldBinlog
,
len
(
plan
.
SegmentBinlogs
))
for
_
,
seg
:=
range
plan
.
SegmentBinlogs
{
segIDs
[
seg
.
SegmentID
]
=
seg
.
Deltalogs
}
log
.
Info
(
"time cost of generating global compaction"
,
zap
.
Any
(
"segID2DeltaLogs"
,
segIDs
),
zap
.
Int64
(
"planID"
,
plan
.
PlanID
),
zap
.
Any
(
"time cost"
,
time
.
Since
(
start
)
.
Milliseconds
()),
zap
.
Int64
(
"collectionID"
,
signal
.
collectionID
),
zap
.
String
(
"channel"
,
group
.
channelName
),
zap
.
Int64
(
"partitionID"
,
group
.
partitionID
))
}
}
...
...
internal/datanode/compactor.go
浏览文件 @
c9174d55
...
...
@@ -133,12 +133,12 @@ func (t *compactionTask) getChannelName() string {
return
t
.
plan
.
GetChannel
()
}
func
(
t
*
compactionTask
)
mergeDeltalogs
(
dBlobs
map
[
UniqueID
][]
*
Blob
,
timetravelTs
Timestamp
)
(
map
[
primaryKey
]
Timestamp
,
*
DelDataBuf
,
error
)
{
func
(
t
*
compactionTask
)
mergeDeltalogs
(
dBlobs
map
[
UniqueID
][]
*
Blob
,
timetravelTs
Timestamp
)
(
map
[
interface
{}
]
Timestamp
,
*
DelDataBuf
,
error
)
{
mergeStart
:=
time
.
Now
()
dCodec
:=
storage
.
NewDeleteCodec
()
var
(
pk2ts
=
make
(
map
[
primaryKey
]
Timestamp
)
pk2ts
=
make
(
map
[
interface
{}
]
Timestamp
)
dbuff
=
&
DelDataBuf
{
delData
:
&
DeleteData
{
Pks
:
make
([]
primaryKey
,
0
),
...
...
@@ -162,7 +162,7 @@ func (t *compactionTask) mergeDeltalogs(dBlobs map[UniqueID][]*Blob, timetravelT
ts
:=
dData
.
Tss
[
i
]
if
timetravelTs
!=
Timestamp
(
0
)
&&
dData
.
Tss
[
i
]
<=
timetravelTs
{
pk2ts
[
pk
]
=
ts
pk2ts
[
pk
.
GetValue
()
]
=
ts
continue
}
...
...
@@ -191,7 +191,7 @@ func nano2Milli(nano time.Duration) float64 {
return
float64
(
nano
)
/
float64
(
time
.
Millisecond
)
}
func
(
t
*
compactionTask
)
merge
(
mergeItr
iterator
,
delta
map
[
primaryKey
]
Timestamp
,
schema
*
schemapb
.
CollectionSchema
,
currentTs
Timestamp
)
([]
*
InsertData
,
int64
,
error
)
{
func
(
t
*
compactionTask
)
merge
(
mergeItr
iterator
,
delta
map
[
interface
{}
]
Timestamp
,
schema
*
schemapb
.
CollectionSchema
,
currentTs
Timestamp
)
([]
*
InsertData
,
int64
,
error
)
{
mergeStart
:=
time
.
Now
()
var
(
...
...
@@ -207,12 +207,10 @@ func (t *compactionTask) merge(mergeItr iterator, delta map[primaryKey]Timestamp
)
isDeletedValue
:=
func
(
v
*
storage
.
Value
)
bool
{
for
pk
,
ts
:=
range
delta
{
if
pk
.
EQ
(
v
.
PK
)
&&
uint64
(
v
.
Timestamp
)
<=
ts
{
return
true
}
ts
,
ok
:=
delta
[
v
.
PK
.
GetValue
()]
if
ok
&&
uint64
(
v
.
Timestamp
)
<=
ts
{
return
true
}
return
false
}
...
...
internal/datanode/compactor_test.go
浏览文件 @
c9174d55
...
...
@@ -259,9 +259,8 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
mitr
:=
storage
.
NewMergeIterator
([]
iterator
{
iitr
})
pk
:=
newInt64PrimaryKey
(
1
)
dm
:=
map
[
primaryKey
]
Timestamp
{
pk
:
10000
,
dm
:=
map
[
interface
{}]
Timestamp
{
1
:
10000
,
}
ct
:=
&
compactionTask
{}
...
...
@@ -289,7 +288,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
mitr
:=
storage
.
NewMergeIterator
([]
iterator
{
iitr
})
dm
:=
map
[
primaryKey
]
Timestamp
{}
dm
:=
map
[
interface
{}
]
Timestamp
{}
ct
:=
&
compactionTask
{}
idata
,
numOfRow
,
err
:=
ct
.
merge
(
mitr
,
dm
,
meta
.
GetSchema
(),
ct
.
GetCurrentTime
())
...
...
@@ -311,10 +310,8 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
require
.
NoError
(
t
,
err
)
mitr
:=
storage
.
NewMergeIterator
([]
iterator
{
iitr
})
pk
:=
newInt64PrimaryKey
(
1
)
dm
:=
map
[
primaryKey
]
Timestamp
{
pk
:
10000
,
dm
:=
map
[
interface
{}]
Timestamp
{
1
:
10000
,
}
ct
:=
&
compactionTask
{}
...
...
internal/storage/primary_key.go
浏览文件 @
c9174d55
...
...
@@ -34,6 +34,7 @@ type PrimaryKey interface {
MarshalJSON
()
([]
byte
,
error
)
UnmarshalJSON
(
data
[]
byte
)
error
SetValue
(
interface
{})
error
GetValue
()
interface
{}
Type
()
schemapb
.
DataType
}
...
...
@@ -147,6 +148,10 @@ func (ip *Int64PrimaryKey) Type() schemapb.DataType {
return
schemapb
.
DataType_Int64
}
func
(
ip
*
Int64PrimaryKey
)
GetValue
()
interface
{}
{
return
ip
.
Value
}
type
BaseStringPrimaryKey
struct
{
Value
string
}
...
...
@@ -199,6 +204,10 @@ func (sp *BaseStringPrimaryKey) SetValue(data interface{}) error {
return
nil
}
func
(
sp
*
BaseStringPrimaryKey
)
GetValue
()
interface
{}
{
return
sp
.
Value
}
type
VarCharPrimaryKey
struct
{
BaseStringPrimaryKey
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录