Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
3ef67c04
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
3ef67c04
编写于
12月 03, 2021
作者:
T
ThreadDao
提交者:
GitHub
12月 03, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[skip ci] Update scale query node test (#12729)
Signed-off-by:
N
ThreadDao
<
yufen.zong@zilliz.com
>
上级
a2f6e31d
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
76 addition
and
86 deletion
+76
-86
tests/python_client/scale/test_query_node_scale.py
tests/python_client/scale/test_query_node_scale.py
+76
-86
未找到文件。
tests/python_client/scale/test_query_node_scale.py
浏览文件 @
3ef67c04
import
pdb
import
threading
import
random
import
time
import
pytest
import
pytest
from
base.collection_wrapper
import
ApiCollectionWrapper
from
base.collection_wrapper
import
ApiCollectionWrapper
from
common.common_type
import
CaseLabel
from
common.common_type
import
CaseLabel
from
scale.helm_env
import
HelmEnv
from
customize.milvus_operator
import
MilvusOperator
from
common
import
common_func
as
cf
from
common
import
common_func
as
cf
from
common
import
common_type
as
ct
from
common
import
common_type
as
ct
from
scale
import
constants
from
scale
import
constants
from
pymilvus
import
Index
,
connections
from
pymilvus
import
Index
,
connections
from
utils.util_log
import
test_log
as
log
from
utils.util_k8s
import
wait_pods_ready
prefix
=
"search_scale"
prefix
=
"search_scale"
nb
=
5000
nb
=
5000
...
@@ -22,98 +24,86 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
...
@@ -22,98 +24,86 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
class
TestQueryNodeScale
:
class
TestQueryNodeScale
:
@
pytest
.
mark
.
tags
(
CaseLabel
.
L3
)
@
pytest
.
mark
.
tags
(
CaseLabel
.
L3
)
def
test_
expand
_query_node
(
self
):
def
test_
scale
_query_node
(
self
):
release_name
=
"scale-query"
release_name
=
"scale-query"
env
=
HelmEnv
(
release_name
=
release_name
)
query_config
=
{
host
=
env
.
helm_install_cluster_milvus
()
'metadata.namespace'
:
constants
.
NAMESPACE
,
'metadata.name'
:
release_name
,
'spec.components.image'
:
'harbor.zilliz.cc/milvus/milvus:master-20211202-ed546d0'
,
'spec.components.proxy.serviceType'
:
'LoadBalancer'
,
'spec.components.queryNode.replicas'
:
1
,
'spec.config.dataCoord.enableCompaction'
:
True
,
'spec.config.dataCoord.enableGarbageCollection'
:
True
}
mic
=
MilvusOperator
()
mic
.
install
(
query_config
)
healthy
=
mic
.
wait_for_healthy
(
release_name
,
constants
.
NAMESPACE
,
timeout
=
1200
)
log
.
info
(
f
"milvus healthy:
{
healthy
}
"
)
host
=
mic
.
endpoint
(
release_name
,
constants
.
NAMESPACE
).
split
(
':'
)[
0
]
# host = "10.98.0.8"
# connect
# connect
connections
.
add_connection
(
default
=
{
"host"
:
host
,
"port"
:
19530
})
connections
.
add_connection
(
default
=
{
"host"
:
host
,
"port"
:
19530
})
connections
.
connect
(
alias
=
'default'
)
connections
.
connect
(
alias
=
'default'
)
# create
# create
c_name
=
"query_scale_one"
c_name
=
cf
.
gen_unique_str
(
"scale_query"
)
# c_name = 'scale_query_DymS7kI4'
collection_w
=
ApiCollectionWrapper
()
collection_w
=
ApiCollectionWrapper
()
collection_w
.
init_collection
(
name
=
c_name
,
schema
=
cf
.
gen_default_collection_schema
())
collection_w
.
init_collection
(
name
=
c_name
,
schema
=
cf
.
gen_default_collection_schema
(),
shards_num
=
2
)
# insert
data
=
cf
.
gen_default_list_data
(
ct
.
default_nb
)
mutation_res
,
_
=
collection_w
.
insert
(
data
)
assert
mutation_res
.
insert_count
==
ct
.
default_nb
# # create index
# collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
# assert collection_w.has_index()[0]
# assert collection_w.index()[0] == Index(collection_w.collection, ct.default_float_vec_field_name,
# default_index_params)
collection_w
.
load
()
# vectors = [[random.random() for _ in range(ct.default_dim)] for _ in range(5)]
res1
,
_
=
collection_w
.
search
(
data
[
-
1
][:
5
],
ct
.
default_float_vec_field_name
,
ct
.
default_search_params
,
ct
.
default_limit
)
# scale queryNode pod
env
.
helm_upgrade_cluster_milvus
(
queryNode
=
2
)
c_name_2
=
"query_scale_two"
collection_w2
=
ApiCollectionWrapper
()
collection_w2
.
init_collection
(
name
=
c_name_2
,
schema
=
cf
.
gen_default_collection_schema
())
collection_w2
.
insert
(
data
)
assert
collection_w2
.
num_entities
==
ct
.
default_nb
collection_w2
.
load
()
res2
,
_
=
collection_w2
.
search
(
data
[
-
1
][:
5
],
ct
.
default_float_vec_field_name
,
ct
.
default_search_params
,
ct
.
default_limit
)
assert
res1
[
0
].
ids
==
res2
[
0
].
ids
# insert two segments
for
i
in
range
(
3
):
df
=
cf
.
gen_default_dataframe_data
(
nb
)
collection_w
.
insert
(
df
)
log
.
debug
(
collection_w
.
num_entities
)
@
pytest
.
mark
.
tags
(
CaseLabel
.
L3
)
# create index
def
test_shrink_query_node
(
self
):
collection_w
.
create_index
(
ct
.
default_float_vec_field_name
,
default_index_params
)
"""
assert
collection_w
.
has_index
()[
0
]
target: test shrink queryNode from 2 to 1
assert
collection_w
.
index
()[
0
]
==
Index
(
collection_w
.
collection
,
ct
.
default_float_vec_field_name
,
method: 1.deploy two queryNode
default_index_params
)
2.search two collections in two queryNode
3.upgrade queryNode from 2 to 1
4.search second collection
expected: search result is correct
"""
# deploy
release_name
=
"scale-query"
env
=
HelmEnv
(
release_name
=
release_name
,
queryNode
=
2
)
host
=
env
.
helm_install_cluster_milvus
(
image_pull_policy
=
constants
.
IF_NOT_PRESENT
)
# connect
# load
connections
.
add_connection
(
default
=
{
"host"
:
host
,
"port"
:
19530
})
connections
.
connect
(
alias
=
'default'
)
# collection one
data
=
cf
.
gen_default_list_data
(
nb
)
c_name
=
"query_scale_one"
collection_w
=
ApiCollectionWrapper
()
collection_w
.
init_collection
(
name
=
c_name
,
schema
=
cf
.
gen_default_collection_schema
())
collection_w
.
insert
(
data
)
assert
collection_w
.
num_entities
==
nb
collection_w
.
load
()
collection_w
.
load
()
res1
,
_
=
collection_w
.
search
(
data
[
-
1
][:
nq
],
ct
.
default_float_vec_field_name
,
ct
.
default_search_params
,
ct
.
default_limit
)
# scale queryNode to 5
assert
res1
[
0
].
ids
[
0
]
==
data
[
0
][
0
]
mic
.
upgrade
(
release_name
,
{
'spec.components.queryNode.replicas'
:
5
},
constants
.
NAMESPACE
)
# collection two
# continuously search
c_name_2
=
"query_scale_two"
def
do_search
():
collection_w2
=
ApiCollectionWrapper
()
while
True
:
collection_w2
.
init_collection
(
name
=
c_name_2
,
schema
=
cf
.
gen_default_collection_schema
())
search_res
,
_
=
collection_w
.
search
(
cf
.
gen_vectors
(
1
,
ct
.
default_dim
),
collection_w2
.
insert
(
data
)
ct
.
default_float_vec_field_name
,
assert
collection_w2
.
num_entities
==
nb
ct
.
default_search_params
,
ct
.
default_limit
)
collection_w2
.
load
()
log
.
debug
(
search_res
[
0
].
ids
)
res2
,
_
=
collection_w2
.
search
(
data
[
-
1
][:
nq
],
ct
.
default_float_vec_field_name
,
assert
len
(
search_res
[
0
].
ids
)
==
ct
.
default_limit
ct
.
default_search_params
,
ct
.
default_limit
)
assert
res2
[
0
].
ids
[
0
]
==
data
[
0
][
0
]
t_search
=
threading
.
Thread
(
target
=
do_search
,
args
=
(),
daemon
=
True
)
t_search
.
start
()
# scale queryNode pod
env
.
helm_upgrade_cluster_milvus
(
queryNode
=
1
)
# wait new QN running, continuously insert
# time.sleep(10)
# search
healthy
=
mic
.
wait_for_healthy
(
release_name
,
constants
.
NAMESPACE
,
timeout
=
1200
)
res1
,
_
=
collection_w
.
search
(
data
[
-
1
][:
nq
],
ct
.
default_float_vec_field_name
,
log
.
info
(
f
"milvus healthy after scale up:
{
healthy
}
"
)
ct
.
default_search_params
,
ct
.
default_limit
)
# wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
assert
res1
[
0
].
ids
[
0
]
==
data
[
0
][
0
]
res2
,
_
=
collection_w2
.
search
(
data
[
-
1
][:
nq
],
ct
.
default_float_vec_field_name
,
def
do_insert
():
ct
.
default_search_params
,
ct
.
default_limit
)
while
True
:
assert
res2
[
0
].
ids
[
0
]
==
data
[
0
][
0
]
tmp_df
=
cf
.
gen_default_dataframe_data
(
1000
)
collection_w
.
insert
(
tmp_df
)
# env.helm_uninstall_cluster_milvus()
\ No newline at end of file
t_insert
=
threading
.
Thread
(
target
=
do_insert
,
args
=
(),
daemon
=
True
)
t_insert
.
start
()
log
.
debug
(
collection_w
.
num_entities
)
time
.
sleep
(
20
)
log
.
debug
(
"Expand querynode test finished"
)
mic
.
upgrade
(
release_name
,
{
'spec.components.queryNode.replicas'
:
3
},
constants
.
NAMESPACE
)
time
.
sleep
(
60
)
wait_pods_ready
(
constants
.
NAMESPACE
,
f
"app.kubernetes.io/instance=
{
release_name
}
"
)
log
.
debug
(
collection_w
.
num_entities
)
time
.
sleep
(
60
)
log
.
debug
(
"Shrink querynode test finished"
)
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录