Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
ef0fc37a
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
ef0fc37a
编写于
6月 27, 2022
作者:
C
cai.zhang
提交者:
GitHub
6月 27, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fixed the bug that IndexCoord lost some meta information (#17818)
Signed-off-by:
N
Cai.Zhang
<
cai.zhang@zilliz.com
>
上级
46e0e265
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
194 addition
and
6 deletion
+194
-6
internal/indexcoord/index_coord.go
internal/indexcoord/index_coord.go
+15
-2
internal/indexcoord/index_coord_test.go
internal/indexcoord/index_coord_test.go
+100
-0
internal/indexcoord/meta_table.go
internal/indexcoord/meta_table.go
+9
-4
internal/kv/etcd/embed_etcd_kv.go
internal/kv/etcd/embed_etcd_kv.go
+21
-0
internal/kv/etcd/embed_etcd_kv_test.go
internal/kv/etcd/embed_etcd_kv_test.go
+7
-0
internal/kv/etcd/etcd_kv.go
internal/kv/etcd/etcd_kv.go
+22
-0
internal/kv/etcd/etcd_kv_test.go
internal/kv/etcd/etcd_kv_test.go
+7
-0
internal/kv/kv.go
internal/kv/kv.go
+1
-0
internal/kv/mock_kv.go
internal/kv/mock_kv.go
+4
-0
internal/kv/mock_kv_test.go
internal/kv/mock_kv_test.go
+4
-0
internal/querycoord/replica_test.go
internal/querycoord/replica_test.go
+4
-0
未找到文件。
internal/indexcoord/index_coord.go
浏览文件 @
ef0fc37a
...
...
@@ -28,6 +28,8 @@ import (
"syscall"
"time"
v3rpc
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
"github.com/golang/protobuf/proto"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3
"go.etcd.io/etcd/client/v3"
...
...
@@ -859,8 +861,7 @@ func (i *IndexCoord) watchMetaLoop() {
defer
i
.
loopWg
.
Done
()
log
.
Debug
(
"IndexCoord watchMetaLoop start"
)
watchChan
:=
i
.
metaTable
.
client
.
WatchWithPrefix
(
indexFilePrefix
)
watchChan
:=
i
.
metaTable
.
client
.
WatchWithRevision
(
indexFilePrefix
,
i
.
metaTable
.
revision
)
for
{
select
{
case
<-
ctx
.
Done
()
:
...
...
@@ -871,6 +872,18 @@ func (i *IndexCoord) watchMetaLoop() {
return
}
if
err
:=
resp
.
Err
();
err
!=
nil
{
if
err
==
v3rpc
.
ErrCompacted
{
newMetaTable
,
err
:=
NewMetaTable
(
i
.
metaTable
.
client
)
if
err
!=
nil
{
log
.
Error
(
"Constructing new meta table fails when etcd has a compaction error"
,
zap
.
String
(
"path"
,
indexFilePrefix
),
zap
.
String
(
"etcd error"
,
err
.
Error
()),
zap
.
Error
(
err
))
panic
(
"failed to handle etcd request, exit.."
)
}
i
.
metaTable
=
newMetaTable
i
.
loopWg
.
Add
(
1
)
go
i
.
watchMetaLoop
()
return
}
log
.
Error
(
"received error event from etcd watcher"
,
zap
.
String
(
"path"
,
indexFilePrefix
),
zap
.
Error
(
err
))
panic
(
"failed to handle etcd request, exit.."
)
}
...
...
internal/indexcoord/index_coord_test.go
浏览文件 @
ef0fc37a
...
...
@@ -18,6 +18,7 @@ package indexcoord
import
(
"context"
"fmt"
"math/rand"
"os"
"os/signal"
...
...
@@ -26,6 +27,9 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/internal/kv"
clientv3
"go.etcd.io/etcd/client/v3"
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
...
...
@@ -324,6 +328,102 @@ func TestIndexCoord_watchNodeLoop(t *testing.T) {
assert
.
True
(
t
,
closed
)
}
type
mockEtcdKv
struct
{
kv
.
MetaKv
watchWithRevision
func
(
string
,
int64
)
clientv3
.
WatchChan
loadWithRevisionAndVersions
func
(
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
}
func
(
mek
*
mockEtcdKv
)
WatchWithRevision
(
key
string
,
revision
int64
)
clientv3
.
WatchChan
{
return
mek
.
watchWithRevision
(
key
,
revision
)
}
func
(
mek
*
mockEtcdKv
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
return
mek
.
loadWithRevisionAndVersions
(
key
)
}
func
TestIndexCoord_watchMetaLoop
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
ic
:=
&
IndexCoord
{
loopCtx
:
ctx
,
loopWg
:
sync
.
WaitGroup
{},
}
watchChan
:=
make
(
chan
clientv3
.
WatchResponse
,
1024
)
client
:=
&
mockEtcdKv
{
watchWithRevision
:
func
(
s
string
,
i
int64
)
clientv3
.
WatchChan
{
return
watchChan
},
}
mt
:=
&
metaTable
{
client
:
client
,
indexBuildID2Meta
:
map
[
UniqueID
]
Meta
{},
revision
:
0
,
lock
:
sync
.
RWMutex
{},
}
ic
.
metaTable
=
mt
t
.
Run
(
"watch chan panic"
,
func
(
t
*
testing
.
T
)
{
ic
.
loopWg
.
Add
(
1
)
watchChan
<-
clientv3
.
WatchResponse
{
Canceled
:
true
}
assert
.
Panics
(
t
,
func
()
{
ic
.
watchMetaLoop
()
})
ic
.
loopWg
.
Wait
()
})
t
.
Run
(
"watch chan new meta table panic"
,
func
(
t
*
testing
.
T
)
{
client
=
&
mockEtcdKv
{
watchWithRevision
:
func
(
s
string
,
i
int64
)
clientv3
.
WatchChan
{
return
watchChan
},
loadWithRevisionAndVersions
:
func
(
s
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
return
[]
string
{},
[]
string
{},
[]
int64
{},
0
,
fmt
.
Errorf
(
"error occurred"
)
},
}
mt
=
&
metaTable
{
client
:
client
,
indexBuildID2Meta
:
map
[
UniqueID
]
Meta
{},
revision
:
0
,
lock
:
sync
.
RWMutex
{},
}
ic
.
metaTable
=
mt
ic
.
loopWg
.
Add
(
1
)
watchChan
<-
clientv3
.
WatchResponse
{
CompactRevision
:
10
}
assert
.
Panics
(
t
,
func
()
{
ic
.
watchMetaLoop
()
})
ic
.
loopWg
.
Wait
()
})
t
.
Run
(
"watch chan new meta success"
,
func
(
t
*
testing
.
T
)
{
ic
.
loopWg
=
sync
.
WaitGroup
{}
client
=
&
mockEtcdKv
{
watchWithRevision
:
func
(
s
string
,
i
int64
)
clientv3
.
WatchChan
{
return
watchChan
},
loadWithRevisionAndVersions
:
func
(
s
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
return
[]
string
{},
[]
string
{},
[]
int64
{},
0
,
nil
},
}
mt
=
&
metaTable
{
client
:
client
,
indexBuildID2Meta
:
map
[
UniqueID
]
Meta
{},
revision
:
0
,
lock
:
sync
.
RWMutex
{},
}
ic
.
metaTable
=
mt
ic
.
loopWg
.
Add
(
1
)
watchChan
<-
clientv3
.
WatchResponse
{
CompactRevision
:
10
}
go
ic
.
watchMetaLoop
()
cancel
()
ic
.
loopWg
.
Wait
()
})
}
func
TestIndexCoord_GetComponentStates
(
t
*
testing
.
T
)
{
n
:=
&
IndexCoord
{}
n
.
stateCode
.
Store
(
internalpb
.
StateCode_Healthy
)
...
...
internal/indexcoord/meta_table.go
浏览文件 @
ef0fc37a
...
...
@@ -25,12 +25,13 @@ import (
"strconv"
"sync"
"github.com/milvus-io/milvus/internal/kv"
"github.com/milvus-io/milvus/internal/metrics"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
etcdkv
"github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
...
...
@@ -47,14 +48,16 @@ type Meta struct {
// metaTable records the mapping of IndexBuildID to Meta.
type
metaTable
struct
{
client
*
etcdkv
.
EtcdKV
// client of a reliable kv service, i.e. etcd client
client
kv
.
MetaKv
// client of a reliable kv service, i.e. etcd client
indexBuildID2Meta
map
[
UniqueID
]
Meta
// index build id to index meta
revision
int64
lock
sync
.
RWMutex
}
// NewMetaTable is used to create a new meta table.
func
NewMetaTable
(
kv
*
etcdkv
.
EtcdKV
)
(
*
metaTable
,
error
)
{
func
NewMetaTable
(
kv
kv
.
MetaKv
)
(
*
metaTable
,
error
)
{
mt
:=
&
metaTable
{
client
:
kv
,
lock
:
sync
.
RWMutex
{},
...
...
@@ -73,11 +76,13 @@ func (mt *metaTable) reloadFromKV() error {
key
:=
indexFilePrefix
log
.
Debug
(
"IndexCoord metaTable LoadWithPrefix "
,
zap
.
String
(
"prefix"
,
key
))
_
,
values
,
versions
,
err
:=
mt
.
client
.
LoadWithPrefix2
(
key
)
_
,
values
,
versions
,
revision
,
err
:=
mt
.
client
.
LoadWithRevisionAndVersions
(
key
)
if
err
!=
nil
{
return
err
}
mt
.
revision
=
revision
for
i
:=
0
;
i
<
len
(
values
);
i
++
{
indexMeta
:=
indexpb
.
IndexMeta
{}
err
=
proto
.
Unmarshal
([]
byte
(
values
[
i
]),
&
indexMeta
)
...
...
internal/kv/etcd/embed_etcd_kv.go
浏览文件 @
ef0fc37a
...
...
@@ -147,6 +147,27 @@ func (kv *EmbedEtcdKV) LoadWithPrefix2(key string) ([]string, []string, []int64,
return
keys
,
values
,
versions
,
nil
}
func
(
kv
*
EmbedEtcdKV
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
key
=
path
.
Join
(
kv
.
rootPath
,
key
)
log
.
Debug
(
"LoadWithPrefix "
,
zap
.
String
(
"prefix"
,
key
))
ctx
,
cancel
:=
context
.
WithTimeout
(
context
.
TODO
(),
RequestTimeout
)
defer
cancel
()
resp
,
err
:=
kv
.
client
.
Get
(
ctx
,
key
,
clientv3
.
WithPrefix
(),
clientv3
.
WithSort
(
clientv3
.
SortByKey
,
clientv3
.
SortAscend
))
if
err
!=
nil
{
return
nil
,
nil
,
nil
,
0
,
err
}
keys
:=
make
([]
string
,
0
,
resp
.
Count
)
values
:=
make
([]
string
,
0
,
resp
.
Count
)
versions
:=
make
([]
int64
,
0
,
resp
.
Count
)
for
_
,
kv
:=
range
resp
.
Kvs
{
keys
=
append
(
keys
,
string
(
kv
.
Key
))
values
=
append
(
values
,
string
(
kv
.
Value
))
versions
=
append
(
versions
,
kv
.
Version
)
}
return
keys
,
values
,
versions
,
resp
.
Header
.
Revision
,
nil
}
// LoadBytesWithPrefix2 returns all the keys and values with versions by the given key prefix
func
(
kv
*
EmbedEtcdKV
)
LoadBytesWithPrefix2
(
key
string
)
([]
string
,
[][]
byte
,
[]
int64
,
error
)
{
key
=
path
.
Join
(
kv
.
rootPath
,
key
)
...
...
internal/kv/etcd/embed_etcd_kv_test.go
浏览文件 @
ef0fc37a
...
...
@@ -128,6 +128,13 @@ func TestEmbedEtcd(te *testing.T) {
assert
.
ElementsMatch
(
t
,
test
.
expectedValues
,
actualValues
)
assert
.
NotZero
(
t
,
versions
)
assert
.
Equal
(
t
,
test
.
expectedError
,
err
)
actualKeys
,
actualValues
,
versions
,
revision
,
err
:=
metaKv
.
LoadWithRevisionAndVersions
(
test
.
prefix
)
assert
.
ElementsMatch
(
t
,
test
.
expectedKeys
,
actualKeys
)
assert
.
ElementsMatch
(
t
,
test
.
expectedValues
,
actualValues
)
assert
.
NotZero
(
t
,
versions
)
assert
.
NotZero
(
t
,
revision
)
assert
.
Equal
(
t
,
test
.
expectedError
,
err
)
}
removeTests
:=
[]
struct
{
...
...
internal/kv/etcd/etcd_kv.go
浏览文件 @
ef0fc37a
...
...
@@ -124,6 +124,28 @@ func (kv *EtcdKV) LoadWithPrefix2(key string) ([]string, []string, []int64, erro
return
keys
,
values
,
versions
,
nil
}
func
(
kv
*
EtcdKV
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
start
:=
time
.
Now
()
key
=
path
.
Join
(
kv
.
rootPath
,
key
)
ctx
,
cancel
:=
context
.
WithTimeout
(
context
.
TODO
(),
RequestTimeout
)
defer
cancel
()
resp
,
err
:=
kv
.
client
.
Get
(
ctx
,
key
,
clientv3
.
WithPrefix
(),
clientv3
.
WithSort
(
clientv3
.
SortByKey
,
clientv3
.
SortAscend
))
if
err
!=
nil
{
return
nil
,
nil
,
nil
,
0
,
err
}
keys
:=
make
([]
string
,
0
,
resp
.
Count
)
values
:=
make
([]
string
,
0
,
resp
.
Count
)
versions
:=
make
([]
int64
,
0
,
resp
.
Count
)
for
_
,
kv
:=
range
resp
.
Kvs
{
keys
=
append
(
keys
,
string
(
kv
.
Key
))
values
=
append
(
values
,
string
(
kv
.
Value
))
versions
=
append
(
versions
,
kv
.
Version
)
}
CheckElapseAndWarn
(
start
,
"Slow etcd operation load with prefix2"
)
return
keys
,
values
,
versions
,
resp
.
Header
.
Revision
,
nil
}
// LoadBytesWithPrefix2 returns all the the keys,values and key versions with the given key prefix.
func
(
kv
*
EtcdKV
)
LoadBytesWithPrefix2
(
key
string
)
([]
string
,
[][]
byte
,
[]
int64
,
error
)
{
start
:=
time
.
Now
()
...
...
internal/kv/etcd/etcd_kv_test.go
浏览文件 @
ef0fc37a
...
...
@@ -126,6 +126,13 @@ func TestEtcdKV_Load(te *testing.T) {
assert
.
ElementsMatch
(
t
,
test
.
expectedValues
,
actualValues
)
assert
.
NotZero
(
t
,
versions
)
assert
.
Equal
(
t
,
test
.
expectedError
,
err
)
actualKeys
,
actualValues
,
versions
,
revision
,
err
:=
etcdKV
.
LoadWithRevisionAndVersions
(
test
.
prefix
)
assert
.
ElementsMatch
(
t
,
test
.
expectedKeys
,
actualKeys
)
assert
.
ElementsMatch
(
t
,
test
.
expectedValues
,
actualValues
)
assert
.
NotZero
(
t
,
versions
)
assert
.
NotZero
(
t
,
revision
)
assert
.
Equal
(
t
,
test
.
expectedError
,
err
)
}
removeTests
:=
[]
struct
{
...
...
internal/kv/kv.go
浏览文件 @
ef0fc37a
...
...
@@ -64,6 +64,7 @@ type MetaKv interface {
GetPath
(
key
string
)
string
LoadWithPrefix
(
key
string
)
([]
string
,
[]
string
,
error
)
LoadWithPrefix2
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
error
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
LoadWithRevision
(
key
string
)
([]
string
,
[]
string
,
int64
,
error
)
Watch
(
key
string
)
clientv3
.
WatchChan
WatchWithPrefix
(
key
string
)
clientv3
.
WatchChan
...
...
internal/kv/mock_kv.go
浏览文件 @
ef0fc37a
...
...
@@ -107,6 +107,10 @@ func (m *MockMetaKV) LoadWithPrefix2(key string) ([]string, []string, []int64, e
panic
(
"not implemented"
)
// TODO: Implement
}
func
(
m
*
MockMetaKV
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
panic
(
"not implemented"
)
// TODO: Implement
}
func
(
m
*
MockMetaKV
)
LoadWithRevision
(
key
string
)
([]
string
,
[]
string
,
int64
,
error
)
{
panic
(
"not implemented"
)
// TODO: Implement
}
...
...
internal/kv/mock_kv_test.go
浏览文件 @
ef0fc37a
...
...
@@ -90,6 +90,10 @@ func TestMockKV_MetaKV(t *testing.T) {
mockKv
.
LoadWithPrefix2
(
testKey
)
})
assert
.
Panics
(
t
,
func
()
{
mockKv
.
LoadWithRevisionAndVersions
(
testKey
)
})
assert
.
Panics
(
t
,
func
()
{
mockKv
.
LoadWithRevision
(
testKey
)
})
...
...
internal/querycoord/replica_test.go
浏览文件 @
ef0fc37a
...
...
@@ -89,6 +89,10 @@ func (m *mockMetaKV) LoadWithPrefix2(key string) ([]string, []string, []int64, e
panic
(
"not implemented"
)
// TODO: Implement
}
func
(
m
*
mockMetaKV
)
LoadWithRevisionAndVersions
(
key
string
)
([]
string
,
[]
string
,
[]
int64
,
int64
,
error
)
{
panic
(
"not implemented"
)
// TODO: Implement
}
func
(
m
*
mockMetaKV
)
LoadWithRevision
(
key
string
)
([]
string
,
[]
string
,
int64
,
error
)
{
panic
(
"not implemented"
)
// TODO: Implement
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录