Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7ac1821c
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7ac1821c
编写于
2月 20, 2021
作者:
B
BossZou
提交者:
yefu.chen
2月 20, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add RocksMQ throughout test
Signed-off-by:
N
BossZou
<
yinghao.zou@zilliz.com
>
上级
cfe139df
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
174 addition
and
7 deletion
+174
-7
internal/distributed/proxynode/service.go
internal/distributed/proxynode/service.go
+4
-4
internal/msgstream/pulsarms/msg_test.go
internal/msgstream/pulsarms/msg_test.go
+5
-1
internal/proxynode/impl.go
internal/proxynode/impl.go
+26
-0
internal/proxynode/interface.go
internal/proxynode/interface.go
+2
-0
internal/proxynode/proxy_node.go
internal/proxynode/proxy_node.go
+1
-1
internal/proxynode/task.go
internal/proxynode/task.go
+69
-0
internal/util/rocksmq/rocksmq_test.go
internal/util/rocksmq/rocksmq_test.go
+66
-0
tests/python/test_index.py
tests/python/test_index.py
+1
-1
未找到文件。
internal/distributed/proxynode/service.go
浏览文件 @
7ac1821c
...
@@ -48,10 +48,6 @@ type Server struct {
...
@@ -48,10 +48,6 @@ type Server struct {
indexServiceClient
*
grpcindexserviceclient
.
Client
indexServiceClient
*
grpcindexserviceclient
.
Client
}
}
func
(
s
*
Server
)
DropIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
panic
(
"implement me"
)
}
func
NewServer
(
ctx
context
.
Context
,
factory
msgstream
.
Factory
)
(
*
Server
,
error
)
{
func
NewServer
(
ctx
context
.
Context
,
factory
msgstream
.
Factory
)
(
*
Server
,
error
)
{
server
:=
&
Server
{
server
:=
&
Server
{
...
@@ -307,6 +303,10 @@ func (s *Server) CreateIndex(ctx context.Context, request *milvuspb.CreateIndexR
...
@@ -307,6 +303,10 @@ func (s *Server) CreateIndex(ctx context.Context, request *milvuspb.CreateIndexR
return
s
.
impl
.
CreateIndex
(
request
)
return
s
.
impl
.
CreateIndex
(
request
)
}
}
func
(
s
*
Server
)
DropIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
s
.
impl
.
DropIndex
(
request
)
}
func
(
s
*
Server
)
DescribeIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
{
func
(
s
*
Server
)
DescribeIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
{
return
s
.
impl
.
DescribeIndex
(
request
)
return
s
.
impl
.
DescribeIndex
(
request
)
}
}
...
...
internal/msgstream/pulsarms/msg_test.go
浏览文件 @
7ac1821c
...
@@ -164,7 +164,11 @@ func TestStream_task_Insert(t *testing.T) {
...
@@ -164,7 +164,11 @@ func TestStream_task_Insert(t *testing.T) {
msgs
:=
result
.
Msgs
msgs
:=
result
.
Msgs
for
_
,
v
:=
range
msgs
{
for
_
,
v
:=
range
msgs
{
receiveCount
++
receiveCount
++
fmt
.
Println
(
"msg type: "
,
v
.
Type
(),
", msg value: "
,
v
,
"msg tag: "
,
v
.
(
*
InsertTask
)
.
Tag
)
// variable v could be type of '*msgstream.TimeTickMsg', here need to check
// if type conversation is successful
if
task
,
ok
:=
v
.
(
*
InsertTask
);
ok
{
fmt
.
Println
(
"msg type: "
,
v
.
Type
(),
", msg value: "
,
v
,
"msg tag: "
,
task
.
Tag
)
}
}
}
}
}
if
receiveCount
>=
len
(
msgPack
.
Msgs
)
{
if
receiveCount
>=
len
(
msgPack
.
Msgs
)
{
...
...
internal/proxynode/impl.go
浏览文件 @
7ac1821c
...
@@ -540,6 +540,32 @@ func (node *NodeImpl) DescribeIndex(request *milvuspb.DescribeIndexRequest) (*mi
...
@@ -540,6 +540,32 @@ func (node *NodeImpl) DescribeIndex(request *milvuspb.DescribeIndexRequest) (*mi
return
dit
.
result
,
nil
return
dit
.
result
,
nil
}
}
func
(
node
*
NodeImpl
)
DropIndex
(
request
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
{
log
.
Println
(
"Drop index for: "
,
request
)
ctx
,
cancel
:=
context
.
WithTimeout
(
context
.
Background
(),
reqTimeoutInterval
)
defer
cancel
()
dit
:=
&
DropIndexTask
{
Condition
:
NewTaskCondition
(
ctx
),
DropIndexRequest
:
request
,
masterClient
:
node
.
masterClient
,
}
err
:=
node
.
sched
.
DdQueue
.
Enqueue
(
dit
)
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
err
.
Error
(),
},
nil
}
err
=
dit
.
WaitToFinish
()
if
err
!=
nil
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UNEXPECTED_ERROR
,
Reason
:
err
.
Error
(),
},
nil
}
return
dit
.
result
,
nil
}
func
(
node
*
NodeImpl
)
GetIndexState
(
request
*
milvuspb
.
IndexStateRequest
)
(
*
milvuspb
.
IndexStateResponse
,
error
)
{
func
(
node
*
NodeImpl
)
GetIndexState
(
request
*
milvuspb
.
IndexStateRequest
)
(
*
milvuspb
.
IndexStateResponse
,
error
)
{
// log.Println("Describe index progress for: ", request)
// log.Println("Describe index progress for: ", request)
ctx
,
cancel
:=
context
.
WithTimeout
(
context
.
Background
(),
reqTimeoutInterval
)
ctx
,
cancel
:=
context
.
WithTimeout
(
context
.
Background
(),
reqTimeoutInterval
)
...
...
internal/proxynode/interface.go
浏览文件 @
7ac1821c
...
@@ -22,6 +22,7 @@ type MasterClient interface {
...
@@ -22,6 +22,7 @@ type MasterClient interface {
ShowPartitions
(
in
*
milvuspb
.
ShowPartitionRequest
)
(
*
milvuspb
.
ShowPartitionResponse
,
error
)
ShowPartitions
(
in
*
milvuspb
.
ShowPartitionRequest
)
(
*
milvuspb
.
ShowPartitionResponse
,
error
)
CreateIndex
(
in
*
milvuspb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
CreateIndex
(
in
*
milvuspb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
DescribeIndex
(
in
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
DescribeIndex
(
in
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
DropIndex
(
in
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
ShowSegments
(
in
*
milvuspb
.
ShowSegmentRequest
)
(
*
milvuspb
.
ShowSegmentResponse
,
error
)
ShowSegments
(
in
*
milvuspb
.
ShowSegmentRequest
)
(
*
milvuspb
.
ShowSegmentResponse
,
error
)
DescribeSegment
(
in
*
milvuspb
.
DescribeSegmentRequest
)
(
*
milvuspb
.
DescribeSegmentResponse
,
error
)
DescribeSegment
(
in
*
milvuspb
.
DescribeSegmentRequest
)
(
*
milvuspb
.
DescribeSegmentResponse
,
error
)
}
}
...
@@ -90,6 +91,7 @@ type ProxyNode interface {
...
@@ -90,6 +91,7 @@ type ProxyNode interface {
CreateIndex
(
request
*
milvuspb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
CreateIndex
(
request
*
milvuspb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
DescribeIndex
(
request
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
DescribeIndex
(
request
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
GetIndexState
(
request
*
milvuspb
.
IndexStateRequest
)
(
*
milvuspb
.
IndexStateResponse
,
error
)
GetIndexState
(
request
*
milvuspb
.
IndexStateRequest
)
(
*
milvuspb
.
IndexStateResponse
,
error
)
DropIndex
(
request
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
Insert
(
request
*
milvuspb
.
InsertRequest
)
(
*
milvuspb
.
InsertResponse
,
error
)
Insert
(
request
*
milvuspb
.
InsertRequest
)
(
*
milvuspb
.
InsertResponse
,
error
)
Search
(
request
*
milvuspb
.
SearchRequest
)
(
*
milvuspb
.
SearchResults
,
error
)
Search
(
request
*
milvuspb
.
SearchRequest
)
(
*
milvuspb
.
SearchResults
,
error
)
...
...
internal/proxynode/proxy_node.go
浏览文件 @
7ac1821c
...
@@ -93,7 +93,7 @@ func (node *NodeImpl) waitForServiceReady(service Component, serviceName string)
...
@@ -93,7 +93,7 @@ func (node *NodeImpl) waitForServiceReady(service Component, serviceName string)
return
nil
return
nil
}
}
// wait for 10 seconds
// wait for 10 seconds
err
:=
retry
.
Retry
(
10
,
time
.
Second
,
checkFunc
)
err
:=
retry
.
Retry
(
10
,
time
.
Millisecond
*
200
,
checkFunc
)
if
err
!=
nil
{
if
err
!=
nil
{
errMsg
:=
fmt
.
Sprintf
(
"ProxyNode wait for %s ready failed"
,
serviceName
)
errMsg
:=
fmt
.
Sprintf
(
"ProxyNode wait for %s ready failed"
,
serviceName
)
return
errors
.
New
(
errMsg
)
return
errors
.
New
(
errMsg
)
...
...
internal/proxynode/task.go
浏览文件 @
7ac1821c
...
@@ -1350,6 +1350,75 @@ func (dit *DescribeIndexTask) PostExecute() error {
...
@@ -1350,6 +1350,75 @@ func (dit *DescribeIndexTask) PostExecute() error {
return
nil
return
nil
}
}
type
DropIndexTask
struct
{
Condition
*
milvuspb
.
DropIndexRequest
masterClient
MasterClient
result
*
commonpb
.
Status
}
func
(
dit
*
DropIndexTask
)
OnEnqueue
()
error
{
dit
.
Base
=
&
commonpb
.
MsgBase
{}
return
nil
}
func
(
dit
*
DropIndexTask
)
ID
()
UniqueID
{
return
dit
.
Base
.
MsgID
}
func
(
dit
*
DropIndexTask
)
SetID
(
uid
UniqueID
)
{
dit
.
Base
.
MsgID
=
uid
}
func
(
dit
*
DropIndexTask
)
Type
()
commonpb
.
MsgType
{
return
dit
.
Base
.
MsgType
}
func
(
dit
*
DropIndexTask
)
BeginTs
()
Timestamp
{
return
dit
.
Base
.
Timestamp
}
func
(
dit
*
DropIndexTask
)
EndTs
()
Timestamp
{
return
dit
.
Base
.
Timestamp
}
func
(
dit
*
DropIndexTask
)
SetTs
(
ts
Timestamp
)
{
dit
.
Base
.
Timestamp
=
ts
}
func
(
dit
*
DropIndexTask
)
PreExecute
()
error
{
dit
.
Base
.
MsgType
=
commonpb
.
MsgType_kDropIndex
dit
.
Base
.
SourceID
=
Params
.
ProxyID
collName
,
fieldName
:=
dit
.
CollectionName
,
dit
.
FieldName
if
err
:=
ValidateCollectionName
(
collName
);
err
!=
nil
{
return
err
}
if
err
:=
ValidateFieldName
(
fieldName
);
err
!=
nil
{
return
err
}
return
nil
}
func
(
dit
*
DropIndexTask
)
Execute
()
error
{
var
err
error
dit
.
result
,
err
=
dit
.
masterClient
.
DropIndex
(
dit
.
DropIndexRequest
)
if
dit
.
result
==
nil
{
return
errors
.
New
(
"drop index resp is nil"
)
}
if
dit
.
result
.
ErrorCode
!=
commonpb
.
ErrorCode_SUCCESS
{
return
errors
.
New
(
dit
.
result
.
Reason
)
}
return
err
}
func
(
dit
*
DropIndexTask
)
PostExecute
()
error
{
return
nil
}
type
GetIndexStateTask
struct
{
type
GetIndexStateTask
struct
{
Condition
Condition
*
milvuspb
.
IndexStateRequest
*
milvuspb
.
IndexStateRequest
...
...
internal/util/rocksmq/rocksmq_test.go
浏览文件 @
7ac1821c
package
rocksmq
package
rocksmq
import
(
import
(
"log"
"os"
"os"
"strconv"
"strconv"
"sync"
"sync"
"testing"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/assert"
etcdkv
"github.com/zilliztech/milvus-distributed/internal/kv/etcd"
etcdkv
"github.com/zilliztech/milvus-distributed/internal/kv/etcd"
...
@@ -203,3 +205,67 @@ func TestRocksMQ_Goroutines(t *testing.T) {
...
@@ -203,3 +205,67 @@ func TestRocksMQ_Goroutines(t *testing.T) {
}
}
wg
.
Wait
()
wg
.
Wait
()
}
}
/**
This test is aim to measure RocksMq throughout.
Hardware:
CPU Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
Disk SSD
Test with 1,000,000 message, result is as follow:
Produce: 190000 message / s
Consume: 90000 message / s
*/
func
TestRocksMQ_Throughout
(
t
*
testing
.
T
)
{
etcdAddr
:=
os
.
Getenv
(
"ETCD_ADDRESS"
)
if
etcdAddr
==
""
{
etcdAddr
=
"localhost:2379"
}
cli
,
err
:=
clientv3
.
New
(
clientv3
.
Config
{
Endpoints
:
[]
string
{
etcdAddr
}})
assert
.
Nil
(
t
,
err
)
etcdKV
:=
etcdkv
.
NewEtcdKV
(
cli
,
"/etcd/test/root"
)
defer
etcdKV
.
Close
()
idAllocator
:=
NewGlobalIDAllocator
(
"dummy"
,
etcdKV
)
_
=
idAllocator
.
Initialize
()
name
:=
"/tmp/rocksmq_3"
defer
os
.
RemoveAll
(
name
)
rmq
,
err
:=
NewRocksMQ
(
name
,
idAllocator
)
assert
.
Nil
(
t
,
err
)
channelName
:=
"channel_throughout_test"
err
=
rmq
.
CreateChannel
(
channelName
)
assert
.
Nil
(
t
,
err
)
defer
rmq
.
DestroyChannel
(
channelName
)
entityNum
:=
1000000
pt0
:=
time
.
Now
()
.
UnixNano
()
/
int64
(
time
.
Millisecond
)
for
i
:=
0
;
i
<
entityNum
;
i
++
{
msg
:=
"message_"
+
strconv
.
Itoa
(
i
)
pMsg
:=
ProducerMessage
{
payload
:
[]
byte
(
msg
)}
assert
.
Nil
(
t
,
idAllocator
.
UpdateID
())
err
:=
rmq
.
Produce
(
channelName
,
[]
ProducerMessage
{
pMsg
})
assert
.
Nil
(
t
,
err
)
}
pt1
:=
time
.
Now
()
.
UnixNano
()
/
int64
(
time
.
Millisecond
)
pDuration
:=
pt1
-
pt0
log
.
Printf
(
"Total produce %d item, cost %v ms, throughout %v / s"
,
entityNum
,
pDuration
,
int64
(
entityNum
)
*
1000
/
pDuration
)
groupName
:=
"test_throughout_group"
_
=
rmq
.
DestroyConsumerGroup
(
groupName
,
channelName
)
_
,
err
=
rmq
.
CreateConsumerGroup
(
groupName
,
channelName
)
assert
.
Nil
(
t
,
err
)
defer
rmq
.
DestroyConsumerGroup
(
groupName
,
channelName
)
// Consume one message in each goroutine
ct0
:=
time
.
Now
()
.
UnixNano
()
/
int64
(
time
.
Millisecond
)
for
i
:=
0
;
i
<
entityNum
;
i
++
{
cMsgs
,
err
:=
rmq
.
Consume
(
groupName
,
channelName
,
1
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
len
(
cMsgs
),
1
)
}
ct1
:=
time
.
Now
()
.
UnixNano
()
/
int64
(
time
.
Millisecond
)
cDuration
:=
ct1
-
ct0
log
.
Printf
(
"Total consume %d item, cost %v ms, throughout %v / s"
,
entityNum
,
cDuration
,
int64
(
entityNum
)
*
1000
/
cDuration
)
}
tests/python/test_index.py
浏览文件 @
7ac1821c
...
@@ -524,6 +524,7 @@ class TestIndexBase:
...
@@ -524,6 +524,7 @@ class TestIndexBase:
connect
.
drop_index
(
collection
,
field_name
)
connect
.
drop_index
(
collection
,
field_name
)
@
pytest
.
mark
.
skip
(
"r0.3-test"
)
class
TestIndexBinary
:
class
TestIndexBinary
:
@
pytest
.
fixture
(
@
pytest
.
fixture
(
scope
=
"function"
,
scope
=
"function"
,
...
@@ -593,7 +594,6 @@ class TestIndexBinary:
...
@@ -593,7 +594,6 @@ class TestIndexBinary:
ids
=
connect
.
bulk_insert
(
binary_collection
,
default_binary_entities
,
partition_tag
=
default_tag
)
ids
=
connect
.
bulk_insert
(
binary_collection
,
default_binary_entities
,
partition_tag
=
default_tag
)
connect
.
create_index
(
binary_collection
,
binary_field_name
,
get_jaccard_index
)
connect
.
create_index
(
binary_collection
,
binary_field_name
,
get_jaccard_index
)
@
pytest
.
mark
.
skip
(
"r0.3-test"
)
@
pytest
.
mark
.
timeout
(
BUILD_TIMEOUT
)
@
pytest
.
mark
.
timeout
(
BUILD_TIMEOUT
)
def
test_create_index_search_with_query_vectors
(
self
,
connect
,
binary_collection
,
get_jaccard_index
,
get_nq
):
def
test_create_index_search_with_query_vectors
(
self
,
connect
,
binary_collection
,
get_jaccard_index
,
get_nq
):
'''
'''
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录