未验证 提交 4bcca1c4 编写于 作者: T ThreadDao 提交者: GitHub

[skip e2e] Update querynode memory chaos case (#14534)

Signed-off-by: NThreadDao <yufen.zong@zilliz.com>
上级 b22b72ac
import random
import threading import threading
from time import sleep, time import time
from time import sleep
import pytest import pytest
import datetime import datetime
...@@ -43,13 +45,13 @@ class TestChaosData: ...@@ -43,13 +45,13 @@ class TestChaosData:
def test_chaos_memory_stress_querynode(self, connection, chaos_yaml): def test_chaos_memory_stress_querynode(self, connection, chaos_yaml):
""" """
target: explore query node behavior after memory stress chaos injected and recovered target: explore query node behavior after memory stress chaos injected and recovered
method: 1. create a collection, insert some data method: 1. Create a collection, insert some data
2. inject memory stress chaos 2. Inject memory stress chaos
3. load collection and search, query 3. Start a threas to load, search and query
4. todo (verify query node response) 4. After chaos duration, check query search success rate
5. delete chaos or chaos finished 5. Delete chaos or chaos finished finally
6. release and reload collection, verify search and query is available expected: 1.If memory is insufficient, querynode is OOMKilled and available after restart
expected: after chaos deleted, load, search and query are all available 2.If memory is sufficient, succ rate of query and search both are 1.0
""" """
c_name = 'chaos_memory_nx6DNW4q' c_name = 'chaos_memory_nx6DNW4q'
collection_w = ApiCollectionWrapper() collection_w = ApiCollectionWrapper()
...@@ -57,22 +59,43 @@ class TestChaosData: ...@@ -57,22 +59,43 @@ class TestChaosData:
log.debug(collection_w.schema) log.debug(collection_w.schema)
log.debug(collection_w._shards_num) log.debug(collection_w._shards_num)
# apply memory stress # apply memory stress chaos
# apply_memory_stress(chaos_yaml) chaos_config = gen_experiment_config(chaos_yaml)
log.debug(chaos_config)
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.debug("chaos injected")
duration = chaos_config.get('spec').get('duration')
duration = duration.replace('h', '*3600+').replace('m', '*60+').replace('s', '*1+') + '+0'
meta_name = chaos_config.get('metadata').get('name')
# wait memory stress # wait memory stress
# sleep(constants.WAIT_PER_OP * 2) sleep(constants.WAIT_PER_OP * 2)
# query # try to do release, load, query and serach in a duration time loop
collection_w.release() try:
collection_w.load() start = time.time()
term_expr = f'{ct.default_int64_field_name} in [0, 1, 999, 99]' while time.time() - start < eval(duration):
for i in range(4): collection_w.release()
t0_query = datetime.datetime.now() collection_w.load()
query_res, _ = collection_w.query(term_expr)
tt_query = datetime.datetime.now() - t0_query term_expr = f'{ct.default_int64_field_name} in {[random.randint(0, 100)]}'
log.info(f"{i} query cost: {tt_query}") query_res, _ = collection_w.query(term_expr)
assert len(query_res) == 4 assert len(query_res) == 1
search_res, _ = collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
log.debug(search_res[0].ids)
assert len(search_res[0].ids) == ct.default_limit
except Exception as e:
raise Exception(str(e))
finally:
chaos_res.delete(meta_name)
@pytest.mark.tags(CaseLabel.L3) @pytest.mark.tags(CaseLabel.L3)
@pytest.mark.parametrize('chaos_yaml', get_chaos_yamls()) @pytest.mark.parametrize('chaos_yaml', get_chaos_yamls())
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册