Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
b597a637
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
b597a637
编写于
1月 29, 2021
作者:
X
XuanYang-cn
提交者:
yefu.chen
1月 29, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add datanode metatable tests
Signed-off-by:
N
XuanYang-cn
<
xuan.yang@zilliz.com
>
上级
305c8142
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
101 addition
and
136 deletion
+101
-136
internal/datanode/allocator.go
internal/datanode/allocator.go
+1
-1
internal/datanode/data_node_test.go
internal/datanode/data_node_test.go
+0
-47
internal/datanode/meta_table.go
internal/datanode/meta_table.go
+64
-55
internal/datanode/meta_table_test.go
internal/datanode/meta_table_test.go
+35
-32
scripts/run_go_unittest.sh
scripts/run_go_unittest.sh
+1
-1
未找到文件。
internal/datanode/allocator.go
浏览文件 @
b597a637
...
...
@@ -24,7 +24,7 @@ func newAllocatorImpl(s MasterServiceInterface) *allocatorImpl {
func
(
alloc
*
allocatorImpl
)
allocID
()
(
UniqueID
,
error
)
{
resp
,
err
:=
alloc
.
masterService
.
AllocID
(
&
masterpb
.
IDRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
commonpb
.
MsgType_k
ShowCollections
,
MsgType
:
commonpb
.
MsgType_k
RequestID
,
MsgID
:
1
,
// GOOSE TODO
Timestamp
:
0
,
// GOOSE TODO
SourceID
:
Params
.
NodeID
,
...
...
internal/datanode/data_node_test.go
浏览文件 @
b597a637
package
datanode
import
(
"context"
"fmt"
"log"
"math/rand"
"os"
"strconv"
"testing"
"time"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
etcdkv
"github.com/zilliztech/milvus-distributed/internal/kv/etcd"
"github.com/zilliztech/milvus-distributed/internal/master"
)
func
makeNewChannelNames
(
names
[]
string
,
suffix
string
)
[]
string
{
...
...
@@ -36,52 +31,10 @@ func refreshChannelNames() {
Params
.
InsertChannelNames
=
makeNewChannelNames
(
Params
.
InsertChannelNames
,
suffix
)
}
func
startMaster
(
ctx
context
.
Context
)
{
master
.
Init
()
etcdAddr
:=
master
.
Params
.
EtcdAddress
metaRootPath
:=
master
.
Params
.
MetaRootPath
etcdCli
,
err
:=
clientv3
.
New
(
clientv3
.
Config
{
Endpoints
:
[]
string
{
etcdAddr
}})
if
err
!=
nil
{
panic
(
err
)
}
_
,
err
=
etcdCli
.
Delete
(
context
.
TODO
(),
metaRootPath
,
clientv3
.
WithPrefix
())
if
err
!=
nil
{
panic
(
err
)
}
masterPort
:=
53101
master
.
Params
.
Port
=
masterPort
svr
,
err
:=
master
.
CreateServer
(
ctx
)
if
err
!=
nil
{
log
.
Print
(
"create server failed"
,
zap
.
Error
(
err
))
}
if
err
:=
svr
.
Run
(
int64
(
master
.
Params
.
Port
));
err
!=
nil
{
log
.
Fatal
(
"run server failed"
,
zap
.
Error
(
err
))
}
fmt
.
Println
(
"Waiting for server!"
,
svr
.
IsServing
())
Params
.
MasterAddress
=
master
.
Params
.
Address
+
":"
+
strconv
.
Itoa
(
masterPort
)
}
func
TestMain
(
m
*
testing
.
M
)
{
Params
.
Init
()
refreshChannelNames
()
const
ctxTimeInMillisecond
=
2000
const
closeWithDeadline
=
true
var
ctx
context
.
Context
if
closeWithDeadline
{
var
cancel
context
.
CancelFunc
d
:=
time
.
Now
()
.
Add
(
ctxTimeInMillisecond
*
time
.
Millisecond
)
ctx
,
cancel
=
context
.
WithDeadline
(
context
.
Background
(),
d
)
defer
cancel
()
}
else
{
ctx
=
context
.
Background
()
}
startMaster
(
ctx
)
exitCode
:=
m
.
Run
()
os
.
Exit
(
exitCode
)
}
...
...
internal/datanode/meta_table.go
浏览文件 @
b597a637
...
...
@@ -12,9 +12,9 @@ import (
)
type
metaTable
struct
{
client
kv
.
Txn
Base
//
client
kv
.
Base
//
segID2FlushMeta
map
[
UniqueID
]
*
datapb
.
SegmentFlushMeta
collID2DdlMeta
map
[
UniqueID
]
*
datapb
.
DDLFlushMeta
// GOOSE TODO: addDDLFlush and has DDLFlush
collID2DdlMeta
map
[
UniqueID
]
*
datapb
.
DDLFlushMeta
lock
sync
.
RWMutex
}
...
...
@@ -36,24 +36,6 @@ func NewMetaTable(kv kv.TxnBase) (*metaTable, error) {
return
mt
,
nil
}
func
(
mt
*
metaTable
)
AppendDDLBinlogPaths
(
collID
UniqueID
,
paths
[]
string
)
error
{
mt
.
lock
.
Lock
()
defer
mt
.
lock
.
Unlock
()
_
,
ok
:=
mt
.
collID2DdlMeta
[
collID
]
if
!
ok
{
mt
.
collID2DdlMeta
[
collID
]
=
&
datapb
.
DDLFlushMeta
{
CollectionID
:
collID
,
BinlogPaths
:
make
([]
string
,
0
),
}
}
meta
:=
mt
.
collID2DdlMeta
[
collID
]
meta
.
BinlogPaths
=
append
(
meta
.
BinlogPaths
,
paths
...
)
return
mt
.
saveDDLFlushMeta
(
meta
)
}
func
(
mt
*
metaTable
)
AppendSegBinlogPaths
(
segmentID
UniqueID
,
fieldID
int64
,
dataPaths
[]
string
)
error
{
_
,
ok
:=
mt
.
segID2FlushMeta
[
segmentID
]
if
!
ok
{
...
...
@@ -97,31 +79,23 @@ func (mt *metaTable) CompleteFlush(segmentID UniqueID) error {
return
mt
.
saveSegFlushMeta
(
meta
)
}
// metaTable.lock.Lock() before call this function
func
(
mt
*
metaTable
)
saveDDLFlushMeta
(
meta
*
datapb
.
DDLFlushMeta
)
error
{
value
:=
proto
.
MarshalTextString
(
meta
)
mt
.
collID2DdlMeta
[
meta
.
CollectionID
]
=
meta
prefix
:=
path
.
Join
(
Params
.
DDLFlushMetaSubPath
,
strconv
.
FormatInt
(
meta
.
CollectionID
,
10
))
return
mt
.
client
.
Save
(
prefix
,
value
)
}
func
(
mt
*
metaTable
)
reloadSegMetaFromKV
()
error
{
mt
.
segID2FlushMeta
=
make
(
map
[
UniqueID
]
*
datapb
.
SegmentFlushMeta
)
func
(
mt
*
metaTable
)
reloadDdlMetaFromKV
()
error
{
mt
.
collID2DdlMeta
=
make
(
map
[
UniqueID
]
*
datapb
.
DDLFlushMeta
)
_
,
values
,
err
:=
mt
.
client
.
LoadWithPrefix
(
Params
.
DDLFlushMetaSubPath
)
_
,
values
,
err
:=
mt
.
client
.
LoadWithPrefix
(
Params
.
SegFlushMetaSubPath
)
if
err
!=
nil
{
return
err
}
for
_
,
value
:=
range
values
{
ddlMeta
:=
&
datapb
.
DDL
FlushMeta
{}
err
=
proto
.
UnmarshalText
(
value
,
ddl
Meta
)
flushMeta
:=
&
datapb
.
Segment
FlushMeta
{}
err
=
proto
.
UnmarshalText
(
value
,
flush
Meta
)
if
err
!=
nil
{
return
err
}
mt
.
collID2DdlMeta
[
ddlMeta
.
CollectionID
]
=
ddl
Meta
mt
.
segID2FlushMeta
[
flushMeta
.
SegmentID
]
=
flush
Meta
}
return
nil
}
...
...
@@ -135,26 +109,6 @@ func (mt *metaTable) saveSegFlushMeta(meta *datapb.SegmentFlushMeta) error {
return
mt
.
client
.
Save
(
prefix
,
value
)
}
func
(
mt
*
metaTable
)
reloadSegMetaFromKV
()
error
{
mt
.
segID2FlushMeta
=
make
(
map
[
UniqueID
]
*
datapb
.
SegmentFlushMeta
)
_
,
values
,
err
:=
mt
.
client
.
LoadWithPrefix
(
Params
.
SegFlushMetaSubPath
)
if
err
!=
nil
{
return
err
}
for
_
,
value
:=
range
values
{
flushMeta
:=
&
datapb
.
SegmentFlushMeta
{}
err
=
proto
.
UnmarshalText
(
value
,
flushMeta
)
if
err
!=
nil
{
return
err
}
mt
.
segID2FlushMeta
[
flushMeta
.
SegmentID
]
=
flushMeta
}
return
nil
}
func
(
mt
*
metaTable
)
addSegmentFlush
(
segmentID
UniqueID
)
error
{
mt
.
lock
.
Lock
()
defer
mt
.
lock
.
Unlock
()
...
...
@@ -197,6 +151,61 @@ func (mt *metaTable) getSegBinlogPaths(segmentID UniqueID) (map[int64][]string,
return
ret
,
nil
}
// --- DDL ---
func
(
mt
*
metaTable
)
AppendDDLBinlogPaths
(
collID
UniqueID
,
paths
[]
string
)
error
{
mt
.
lock
.
Lock
()
defer
mt
.
lock
.
Unlock
()
_
,
ok
:=
mt
.
collID2DdlMeta
[
collID
]
if
!
ok
{
mt
.
collID2DdlMeta
[
collID
]
=
&
datapb
.
DDLFlushMeta
{
CollectionID
:
collID
,
BinlogPaths
:
make
([]
string
,
0
),
}
}
meta
:=
mt
.
collID2DdlMeta
[
collID
]
meta
.
BinlogPaths
=
append
(
meta
.
BinlogPaths
,
paths
...
)
return
mt
.
saveDDLFlushMeta
(
meta
)
}
func
(
mt
*
metaTable
)
hasDDLFlushMeta
(
collID
UniqueID
)
bool
{
mt
.
lock
.
RLock
()
defer
mt
.
lock
.
RUnlock
()
_
,
ok
:=
mt
.
collID2DdlMeta
[
collID
]
return
ok
}
// metaTable.lock.Lock() before call this function
func
(
mt
*
metaTable
)
saveDDLFlushMeta
(
meta
*
datapb
.
DDLFlushMeta
)
error
{
value
:=
proto
.
MarshalTextString
(
meta
)
mt
.
collID2DdlMeta
[
meta
.
CollectionID
]
=
meta
prefix
:=
path
.
Join
(
Params
.
DDLFlushMetaSubPath
,
strconv
.
FormatInt
(
meta
.
CollectionID
,
10
))
return
mt
.
client
.
Save
(
prefix
,
value
)
}
func
(
mt
*
metaTable
)
reloadDdlMetaFromKV
()
error
{
mt
.
collID2DdlMeta
=
make
(
map
[
UniqueID
]
*
datapb
.
DDLFlushMeta
)
_
,
values
,
err
:=
mt
.
client
.
LoadWithPrefix
(
Params
.
DDLFlushMetaSubPath
)
if
err
!=
nil
{
return
err
}
for
_
,
value
:=
range
values
{
ddlMeta
:=
&
datapb
.
DDLFlushMeta
{}
err
=
proto
.
UnmarshalText
(
value
,
ddlMeta
)
if
err
!=
nil
{
return
err
}
mt
.
collID2DdlMeta
[
ddlMeta
.
CollectionID
]
=
ddlMeta
}
return
nil
}
func
(
mt
*
metaTable
)
getDDLBinlogPaths
(
collID
UniqueID
)
(
map
[
UniqueID
][]
string
,
error
)
{
mt
.
lock
.
RLock
()
defer
mt
.
lock
.
RUnlock
()
...
...
internal/datanode/meta_table_test.go
浏览文件 @
b597a637
package
datanode
import
(
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
etcdkv
"github.com/zilliztech/milvus-distributed/internal/kv/etcd"
"go.etcd.io/etcd/clientv3"
memkv
"github.com/zilliztech/milvus-distributed/internal/kv/mem"
)
func
TestMetaTable_
all
(
t
*
testing
.
T
)
{
func
TestMetaTable_
SegmentFlush
(
t
*
testing
.
T
)
{
etcdAddr
:=
Params
.
EtcdAddress
cli
,
err
:=
clientv3
.
New
(
clientv3
.
Config
{
Endpoints
:
[]
string
{
etcdAddr
}})
require
.
NoError
(
t
,
err
)
etcdKV
:=
etcdkv
.
NewEtcdKV
(
cli
,
"/etcd/test/meta/root"
)
_
,
err
=
cli
.
Delete
(
context
.
TODO
(),
"/etcd/test/meta/root"
,
clientv3
.
WithPrefix
())
require
.
NoError
(
t
,
err
)
meta
,
err
:=
NewMetaTable
(
etcdKV
)
kvMock
:=
memkv
.
NewMemoryKV
()
meta
,
err
:=
NewMetaTable
(
kvMock
)
assert
.
NoError
(
t
,
err
)
defer
meta
.
client
.
Close
()
...
...
@@ -65,8 +55,37 @@ func TestMetaTable_all(t *testing.T) {
ret
)
})
t
.
Run
(
"TestMetaTable_CompleteFlush"
,
func
(
t
*
testing
.
T
)
{
var
segmentID
UniqueID
=
401
err
:=
meta
.
addSegmentFlush
(
segmentID
)
assert
.
NoError
(
t
,
err
)
ret
,
err
:=
meta
.
checkFlushComplete
(
segmentID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
false
,
ret
)
meta
.
CompleteFlush
(
segmentID
)
ret
,
err
=
meta
.
checkFlushComplete
(
segmentID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
true
,
ret
)
})
}
func
TestMetaTable_DDLFlush
(
t
*
testing
.
T
)
{
kvMock
:=
memkv
.
NewMemoryKV
()
meta
,
err
:=
NewMetaTable
(
kvMock
)
assert
.
NoError
(
t
,
err
)
defer
meta
.
client
.
Close
()
t
.
Run
(
"TestMetaTable_AppendDDLBinlogPaths"
,
func
(
t
*
testing
.
T
)
{
assert
.
False
(
t
,
meta
.
hasDDLFlushMeta
(
301
))
assert
.
False
(
t
,
meta
.
hasDDLFlushMeta
(
302
))
collID2Paths
:=
map
[
UniqueID
][]
string
{
301
:
{
"a"
,
"b"
,
"c"
},
302
:
{
"c"
,
"b"
,
"a"
},
...
...
@@ -84,24 +103,8 @@ func TestMetaTable_all(t *testing.T) {
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
map
[
UniqueID
][]
string
{
k
:
v
},
ret
)
}
})
t
.
Run
(
"TestMetaTable_CompleteFlush"
,
func
(
t
*
testing
.
T
)
{
var
segmentID
UniqueID
=
401
err
:=
meta
.
addSegmentFlush
(
segmentID
)
assert
.
NoError
(
t
,
err
)
ret
,
err
:=
meta
.
checkFlushComplete
(
segmentID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
false
,
ret
)
meta
.
CompleteFlush
(
segmentID
)
ret
,
err
=
meta
.
checkFlushComplete
(
segmentID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
true
,
ret
)
assert
.
True
(
t
,
meta
.
hasDDLFlushMeta
(
301
))
assert
.
True
(
t
,
meta
.
hasDDLFlushMeta
(
302
))
})
}
scripts/run_go_unittest.sh
浏览文件 @
b597a637
...
...
@@ -18,7 +18,7 @@ go test -race -cover "${MILVUS_DIR}/kv/..." -failfast
# TODO: remove to distributed
#go test -race -cover "${MILVUS_DIR}/proxynode/..." -failfast
#go test -race -cover "${MILVUS_DIR}/writenode/..." -failfast
#
go test -race -cover "${MILVUS_DIR}/datanode/..." -failfast
go
test
-race
-cover
"
${
MILVUS_DIR
}
/datanode/..."
-failfast
#go test -race -cover "${MILVUS_DIR}/master/..." -failfast
#go test -race -cover "${MILVUS_DIR}/indexnode/..." -failfast
#go test -race -cover "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/querynode/..." "${MILVUS_DIR}/storage" "${MILVUS_DIR}/util/..." -failfast
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录