未验证 提交 4bcca1c4 编写于 作者: T ThreadDao 提交者: GitHub

[skip e2e] Update querynode memory chaos case (#14534)

Signed-off-by: NThreadDao <yufen.zong@zilliz.com>
上级 b22b72ac
import random
import threading
from time import sleep, time
import time
from time import sleep
import pytest
import datetime
......@@ -43,13 +45,13 @@ class TestChaosData:
def test_chaos_memory_stress_querynode(self, connection, chaos_yaml):
"""
target: explore query node behavior after memory stress chaos injected and recovered
method: 1. create a collection, insert some data
2. inject memory stress chaos
3. load collection and search, query
4. todo (verify query node response)
5. delete chaos or chaos finished
6. release and reload collection, verify search and query is available
expected: after chaos deleted, load, search and query are all available
method: 1. Create a collection, insert some data
2. Inject memory stress chaos
3. Start a threas to load, search and query
4. After chaos duration, check query search success rate
5. Delete chaos or chaos finished finally
expected: 1.If memory is insufficient, querynode is OOMKilled and available after restart
2.If memory is sufficient, succ rate of query and search both are 1.0
"""
c_name = 'chaos_memory_nx6DNW4q'
collection_w = ApiCollectionWrapper()
......@@ -57,22 +59,43 @@ class TestChaosData:
log.debug(collection_w.schema)
log.debug(collection_w._shards_num)
# apply memory stress
# apply_memory_stress(chaos_yaml)
# apply memory stress chaos
chaos_config = gen_experiment_config(chaos_yaml)
log.debug(chaos_config)
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.debug("chaos injected")
duration = chaos_config.get('spec').get('duration')
duration = duration.replace('h', '*3600+').replace('m', '*60+').replace('s', '*1+') + '+0'
meta_name = chaos_config.get('metadata').get('name')
# wait memory stress
# sleep(constants.WAIT_PER_OP * 2)
# query
collection_w.release()
collection_w.load()
term_expr = f'{ct.default_int64_field_name} in [0, 1, 999, 99]'
for i in range(4):
t0_query = datetime.datetime.now()
query_res, _ = collection_w.query(term_expr)
tt_query = datetime.datetime.now() - t0_query
log.info(f"{i} query cost: {tt_query}")
assert len(query_res) == 4
sleep(constants.WAIT_PER_OP * 2)
# try to do release, load, query and serach in a duration time loop
try:
start = time.time()
while time.time() - start < eval(duration):
collection_w.release()
collection_w.load()
term_expr = f'{ct.default_int64_field_name} in {[random.randint(0, 100)]}'
query_res, _ = collection_w.query(term_expr)
assert len(query_res) == 1
search_res, _ = collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
log.debug(search_res[0].ids)
assert len(search_res[0].ids) == ct.default_limit
except Exception as e:
raise Exception(str(e))
finally:
chaos_res.delete(meta_name)
@pytest.mark.tags(CaseLabel.L3)
@pytest.mark.parametrize('chaos_yaml', get_chaos_yamls())
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册