diff --git a/tests/python_client/chaos/test_chaos_memory_stress.py b/tests/python_client/chaos/test_chaos_memory_stress.py index d320c3ae7940079fef85f3081551027847b2d4f4..8c53cefade2e999475d1fdb470d902f7e17af0e8 100644 --- a/tests/python_client/chaos/test_chaos_memory_stress.py +++ b/tests/python_client/chaos/test_chaos_memory_stress.py @@ -1,5 +1,7 @@ +import random import threading -from time import sleep, time +import time +from time import sleep import pytest import datetime @@ -43,13 +45,13 @@ class TestChaosData: def test_chaos_memory_stress_querynode(self, connection, chaos_yaml): """ target: explore query node behavior after memory stress chaos injected and recovered - method: 1. create a collection, insert some data - 2. inject memory stress chaos - 3. load collection and search, query - 4. todo (verify query node response) - 5. delete chaos or chaos finished - 6. release and reload collection, verify search and query is available - expected: after chaos deleted, load, search and query are all available + method: 1. Create a collection, insert some data + 2. Inject memory stress chaos + 3. Start a threas to load, search and query + 4. After chaos duration, check query search success rate + 5. Delete chaos or chaos finished finally + expected: 1.If memory is insufficient, querynode is OOMKilled and available after restart + 2.If memory is sufficient, succ rate of query and search both are 1.0 """ c_name = 'chaos_memory_nx6DNW4q' collection_w = ApiCollectionWrapper() @@ -57,22 +59,43 @@ class TestChaosData: log.debug(collection_w.schema) log.debug(collection_w._shards_num) - # apply memory stress - # apply_memory_stress(chaos_yaml) - + # apply memory stress chaos + chaos_config = gen_experiment_config(chaos_yaml) + log.debug(chaos_config) + chaos_res = CusResource(kind=chaos_config['kind'], + group=constants.CHAOS_GROUP, + version=constants.CHAOS_VERSION, + namespace=constants.CHAOS_NAMESPACE) + chaos_res.create(chaos_config) + log.debug("chaos injected") + duration = chaos_config.get('spec').get('duration') + duration = duration.replace('h', '*3600+').replace('m', '*60+').replace('s', '*1+') + '+0' + meta_name = chaos_config.get('metadata').get('name') # wait memory stress - # sleep(constants.WAIT_PER_OP * 2) - - # query - collection_w.release() - collection_w.load() - term_expr = f'{ct.default_int64_field_name} in [0, 1, 999, 99]' - for i in range(4): - t0_query = datetime.datetime.now() - query_res, _ = collection_w.query(term_expr) - tt_query = datetime.datetime.now() - t0_query - log.info(f"{i} query cost: {tt_query}") - assert len(query_res) == 4 + sleep(constants.WAIT_PER_OP * 2) + + # try to do release, load, query and serach in a duration time loop + try: + start = time.time() + while time.time() - start < eval(duration): + collection_w.release() + collection_w.load() + + term_expr = f'{ct.default_int64_field_name} in {[random.randint(0, 100)]}' + query_res, _ = collection_w.query(term_expr) + assert len(query_res) == 1 + + search_res, _ = collection_w.search(cf.gen_vectors(1, ct.default_dim), + ct.default_float_vec_field_name, + ct.default_search_params, ct.default_limit) + log.debug(search_res[0].ids) + assert len(search_res[0].ids) == ct.default_limit + + except Exception as e: + raise Exception(str(e)) + + finally: + chaos_res.delete(meta_name) @pytest.mark.tags(CaseLabel.L3) @pytest.mark.parametrize('chaos_yaml', get_chaos_yamls())