From a8f7dc86be44e104777744090ddeee413187e1f7 Mon Sep 17 00:00:00 2001 From: binbin <83755740+binbinlv@users.noreply.github.com> Date: Fri, 27 Aug 2021 12:19:57 +0800 Subject: [PATCH] Update test cases after code change and fix (#7314) Signed-off-by: Binbin Lv --- build/ci/jenkins/Jenkinsfile | 2 +- build/ci/jenkins/NightlyCI.groovy | 5 +- tests/python_client/requirements.txt | 2 +- .../testcases/test_utility_20.py | 93 +++++++++++-------- 4 files changed, 58 insertions(+), 44 deletions(-) diff --git a/build/ci/jenkins/Jenkinsfile b/build/ci/jenkins/Jenkinsfile index 947a5f5a5..2cf557d16 100644 --- a/build/ci/jenkins/Jenkinsfile +++ b/build/ci/jenkins/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { sh "./tests/scripts/export_logs.sh" dir("${env.ARTIFACTS}") { sh "find ./kind -path '*/history/*' -type f | xargs tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz --transform='s:^[^/]*/[^/]*/[^/]*/[^/]*/::g' || true" - if ("${MILVUS_CLIENT}" == "pymilvus-orm") { + if ("${MILVUS_CLIENT}" == "pymilvus") { sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ./tests/pytest_logs --remove-files || true" } archiveArtifacts artifacts: "**.tar.gz", allowEmptyArchive: true diff --git a/build/ci/jenkins/NightlyCI.groovy b/build/ci/jenkins/NightlyCI.groovy index 62d6f8b71..d4c14eb02 100644 --- a/build/ci/jenkins/NightlyCI.groovy +++ b/build/ci/jenkins/NightlyCI.groovy @@ -5,7 +5,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "50 22 * * * " : "" -int total_timeout_minutes = 300 +int total_timeout_minutes = 360 int e2e_timeout_seconds = 4 * 60 * 60 pipeline { @@ -67,6 +67,7 @@ pipeline { def clusterEnabled = "false" if ("${MILVUS_SERVER_TYPE}" == "distributed") { clusterEnabled = "true" + e2e_timeout_seconds = 5 * 60 * 60 } if ("${MILVUS_CLIENT}" == "pymilvus") { @@ -139,7 +140,7 @@ pipeline { sh "./tests/scripts/export_logs.sh" dir("${env.ARTIFACTS}") { sh "find ./kind -path '*/history/*' -type f | xargs tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-e2e-nightly-logs.tar.gz --transform='s:^[^/]*/[^/]*/[^/]*/[^/]*/::g' || true" - if ("${MILVUS_CLIENT}" == "pymilvus-orm") { + if ("${MILVUS_CLIENT}" == "pymilvus") { sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ./tests/pytest_logs --remove-files || true" } archiveArtifacts artifacts: "**.tar.gz", allowEmptyArchive: true diff --git a/tests/python_client/requirements.txt b/tests/python_client/requirements.txt index 73bb27d20..e45e1ba3c 100644 --- a/tests/python_client/requirements.txt +++ b/tests/python_client/requirements.txt @@ -12,7 +12,7 @@ pytest-print==0.2.1 pytest-level==0.1.1 pytest-xdist==2.2.1 # pytest-parallel -pymilvus==2.0.0rc5.dev20 +pymilvus==2.0.0rc5.dev29 pytest-rerunfailures==9.1.1 git+https://github.com/Projectplace/pytest-tags ndg-httpsclient diff --git a/tests/python_client/testcases/test_utility_20.py b/tests/python_client/testcases/test_utility_20.py index 584dda679..8a2899c6d 100644 --- a/tests/python_client/testcases/test_utility_20.py +++ b/tests/python_client/testcases/test_utility_20.py @@ -38,6 +38,10 @@ class TestUtilityParams(TestcaseBase): def get_not_support_metric(self, request): yield request.param + @pytest.fixture(scope="function", params=["metric_type", "metric"]) + def get_support_metric_field(self, request): + yield request.param + """ ****************************************************************** # The followings are invalid cases @@ -183,7 +187,6 @@ class TestUtilityParams(TestcaseBase): "is illegal".format(invalid_vector)}) @pytest.mark.tags(CaseLabel.L1) - @pytest.mark.xfail(reason="issue 7038") def test_calc_distance_left_vector_invalid_value(self, get_invalid_vector_dict): """ target: test calculated distance with invalid vectors @@ -196,7 +199,8 @@ class TestUtilityParams(TestcaseBase): self.utility_wrap.calc_distance(invalid_vector, invalid_vector, check_task=CheckTasks.err_res, check_items={"err_code": 1, - "err_msg": "Left vectors array is empty"}) + "err_msg": "vectors_left value {} " + "is illegal".format(invalid_vector)}) @pytest.mark.tags(CaseLabel.L1) def test_calc_distance_right_vector_invalid_type(self, get_invalid_vector_dict): @@ -217,7 +221,6 @@ class TestUtilityParams(TestcaseBase): "is illegal".format(invalid_vector)}) @pytest.mark.tags(CaseLabel.L1) - @pytest.mark.xfail(reason="issue 7038") def test_calc_distance_right_vector_invalid_value(self, get_invalid_vector_dict): """ target: test calculated distance with invalid vectors @@ -232,11 +235,11 @@ class TestUtilityParams(TestcaseBase): self.utility_wrap.calc_distance(op_l, invalid_vector, check_task=CheckTasks.err_res, check_items={"err_code": 1, - "err_msg": "Cannot calculate distance between " - "vectors with different dimension"}) + "err_msg": "vectors_right value {} " + "is illegal".format(invalid_vector)}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_invalid_metric_type(self, get_invalid_metric_type): + def test_calc_distance_invalid_metric_type(self, get_support_metric_field, get_invalid_metric_type): """ target: test calculated distance with invalid metric method: input invalid metric @@ -247,16 +250,17 @@ class TestUtilityParams(TestcaseBase): vectors_r = cf.gen_vectors(default_nb, default_dim) op_l = {"float_vectors": vectors_l} op_r = {"float_vectors": vectors_r} + metric_field = get_support_metric_field metric = get_invalid_metric_type - params = {"metric": metric} + params = {metric_field: metric} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.err_res, check_items={"err_code": 1, - "err_msg": "params value {} " - "is illegal".format(params)}) + "err_msg": "params value {{'metric': {}}} " + "is illegal".format(metric)}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_invalid_metric_value(self, get_invalid_metric_value): + def test_calc_distance_invalid_metric_value(self, get_support_metric_field, get_invalid_metric_value): """ target: test calculated distance with invalid metric method: input invalid metric @@ -267,15 +271,17 @@ class TestUtilityParams(TestcaseBase): vectors_r = cf.gen_vectors(default_nb, default_dim) op_l = {"float_vectors": vectors_l} op_r = {"float_vectors": vectors_r} + metric_field = get_support_metric_field metric = get_invalid_metric_value - params = {"metric": metric} + params = {metric_field: metric} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.err_res, check_items={"err_code": 1, - "err_msg": "Invalid metric type"}) + "err_msg": "{} metric type is invalid for " + "float vector".format(metric)}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_not_support_metric(self, get_not_support_metric): + def test_calc_distance_not_support_metric(self, get_support_metric_field, get_not_support_metric): """ target: test calculated distance with invalid metric method: input invalid metric @@ -286,15 +292,17 @@ class TestUtilityParams(TestcaseBase): vectors_r = cf.gen_vectors(default_nb, default_dim) op_l = {"float_vectors": vectors_l} op_r = {"float_vectors": vectors_r} + metric_field = get_support_metric_field metric = get_not_support_metric - params = {"metric": metric} + params = {metric_field: metric} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.err_res, check_items={"err_code": 1, - "err_msg": "Invalid metric type"}) + "err_msg": "{} metric type is invalid for " + "float vector".format(metric)}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_invalid_using(self): + def test_calc_distance_invalid_using(self, get_support_metric_field): """ target: test calculated distance with invalid using method: input invalid using @@ -305,7 +313,8 @@ class TestUtilityParams(TestcaseBase): vectors_r = cf.gen_vectors(default_nb, default_dim) op_l = {"float_vectors": vectors_l} op_r = {"float_vectors": vectors_r} - params = {"metric": "L2", "sqrt": True} + metric_field = get_support_metric_field + params = {metric_field: "L2", "sqrt": True} using = "empty" self.utility_wrap.calc_distance(op_l, op_r, params, using=using, check_task=CheckTasks.err_res, @@ -332,7 +341,7 @@ class TestUtilityParams(TestcaseBase): "vectors with different dimension"}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_collection_before_load(self): + def test_calc_distance_collection_before_load(self, get_support_metric_field): """ target: test calculated distance when entities is not ready method: calculate distance before load @@ -347,7 +356,8 @@ class TestUtilityParams(TestcaseBase): "field": default_field_name} op_r = {"ids": insert_ids[middle:], "collection": collection_w.name, "field": default_field_name} - params = {"metric": "L2", "sqrt": True} + metric_field = get_support_metric_field + params = {metric_field: "L2", "sqrt": True} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.err_res, check_items={"err_code": 1, @@ -357,6 +367,10 @@ class TestUtilityParams(TestcaseBase): class TestUtilityBase(TestcaseBase): """ Test case of index interface """ + @pytest.fixture(scope="function", params=["metric_type", "metric"]) + def metric_field(self, request): + yield request.param + @pytest.fixture(scope="function", params=[True, False]) def sqrt(self, request): yield request.param @@ -630,7 +644,7 @@ class TestUtilityBase(TestcaseBase): "vectors_r": vectors_r}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_default_sqrt(self, metric): + def test_calc_distance_default_sqrt(self, metric_field, metric): """ target: test calculated distance with default param method: calculated distance with default sqrt @@ -641,7 +655,7 @@ class TestUtilityBase(TestcaseBase): vectors_r = cf.gen_vectors(default_nb, default_dim) op_l = {"float_vectors": vectors_l} op_r = {"float_vectors": vectors_r} - params = {"metric": metric} + params = {metric_field: metric} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.check_distance, check_items={"vectors_l": vectors_l, @@ -649,7 +663,6 @@ class TestUtilityBase(TestcaseBase): "metric": metric}) @pytest.mark.tags(CaseLabel.L2) - @pytest.mark.xfail(reason="issue 7217") def test_calc_distance_default_metric(self, sqrt): """ target: test calculated distance with default param @@ -669,7 +682,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_binary_metric(self, metric_binary): + def test_calc_distance_binary_metric(self, metric_field, metric_binary): """ target: test calculate distance with binary vectors method: calculate distance between binary vectors @@ -681,7 +694,7 @@ class TestUtilityBase(TestcaseBase): raw_vectors_r, vectors_r = cf.gen_binary_vectors(nb, default_dim) op_l = {"bin_vectors": vectors_l} op_r = {"bin_vectors": vectors_r} - params = {"metric": metric_binary} + params = {metric_field: metric_binary} vectors_l = raw_vectors_l vectors_r = raw_vectors_r self.utility_wrap.calc_distance(op_l, op_r, params, @@ -691,7 +704,7 @@ class TestUtilityBase(TestcaseBase): "metric": metric_binary}) @pytest.mark.tags(CaseLabel.L1) - def test_calc_distance_from_collection_ids(self, metric, sqrt): + def test_calc_distance_from_collection_ids(self, metric_field, metric, sqrt): """ target: test calculated distance from collection entities method: both left and right vectors are from collection @@ -710,7 +723,7 @@ class TestUtilityBase(TestcaseBase): "field": default_field_name} op_r = {"ids": insert_ids[middle:], "collection": collection_w.name, "field": default_field_name} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.check_distance, check_items={"vectors_l": vectors_l, @@ -719,7 +732,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_from_collections(self, metric, sqrt): + def test_calc_distance_from_collections(self, metric_field, metric, sqrt): """ target: test calculated distance between entities from collections method: calculated distance between entities from two collections @@ -736,7 +749,7 @@ class TestUtilityBase(TestcaseBase): "field": default_field_name} op_r = {"ids": insert_ids_1, "collection": collection_w_1.name, "field": default_field_name} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.check_distance, check_items={"vectors_l": vectors_l, @@ -745,7 +758,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_left_vector_and_collection_ids(self, metric, sqrt): + def test_calc_distance_left_vector_and_collection_ids(self, metric_field, metric, sqrt): """ target: test calculated distance from collection entities method: set left vectors as random vectors, right vectors from collection @@ -763,7 +776,7 @@ class TestUtilityBase(TestcaseBase): op_l = {"float_vectors": vectors_l} op_r = {"ids": insert_ids[middle:], "collection": collection_w.name, "field": default_field_name} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.check_distance, check_items={"vectors_l": vectors_l, @@ -772,7 +785,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_right_vector_and_collection_ids(self, metric, sqrt): + def test_calc_distance_right_vector_and_collection_ids(self, metric_field, metric, sqrt): """ target: test calculated distance from collection entities method: set right vectors as random vectors, left vectors from collection @@ -788,7 +801,7 @@ class TestUtilityBase(TestcaseBase): op_l = {"ids": insert_ids[:middle], "collection": collection_w.name, "field": default_field_name} op_r = {"float_vectors": vectors_r} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} self.utility_wrap.calc_distance(op_l, op_r, params, check_task=CheckTasks.check_distance, check_items={"vectors_l": vectors_l, @@ -797,7 +810,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_from_partition_ids(self, metric, sqrt): + def test_calc_distance_from_partition_ids(self, metric_field, metric, sqrt): """ target: test calculated distance from one partition entities method: both left and right vectors are from partition @@ -808,7 +821,7 @@ class TestUtilityBase(TestcaseBase): collection_w, vectors, _, insert_ids = self.init_collection_general(prefix, True, nb, partition_num=1) partitions = collection_w.partitions middle = len(insert_ids) // 2 - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} for i in range(len(partitions)): vectors_l = vectors[i].loc[:, default_field_name] vectors_r = vectors[i].loc[:, default_field_name] @@ -824,7 +837,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_from_partitions(self, metric, sqrt): + def test_calc_distance_from_partitions(self, metric_field, metric, sqrt): """ target: test calculated distance between entities from partitions method: calculate distance between entities from two partitions @@ -835,7 +848,7 @@ class TestUtilityBase(TestcaseBase): collection_w, vectors, _, insert_ids = self.init_collection_general(prefix, True, nb, partition_num=1) partitions = collection_w.partitions middle = len(insert_ids) // 2 - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} vectors_l = vectors[0].loc[:, default_field_name] vectors_r = vectors[1].loc[:, default_field_name] op_l = {"ids": insert_ids[:middle], "collection": collection_w.name, @@ -850,7 +863,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_left_vectors_and_partition_ids(self, metric, sqrt): + def test_calc_distance_left_vectors_and_partition_ids(self, metric_field, metric, sqrt): """ target: test calculated distance between vectors and partition entities method: set left vectors as random vectors, right vectors are entities @@ -863,7 +876,7 @@ class TestUtilityBase(TestcaseBase): partitions = collection_w.partitions vectors_l = cf.gen_vectors(nb // 2, default_dim) op_l = {"float_vectors": vectors_l} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} for i in range(len(partitions)): vectors_r = vectors[i].loc[:, default_field_name] op_r = {"ids": insert_ids[middle:], "collection": collection_w.name, @@ -876,7 +889,7 @@ class TestUtilityBase(TestcaseBase): "sqrt": sqrt}) @pytest.mark.tags(CaseLabel.L2) - def test_calc_distance_right_vectors_and_partition_ids(self, metric, sqrt): + def test_calc_distance_right_vectors_and_partition_ids(self, metric_field, metric, sqrt): """ target: test calculated distance between vectors and partition entities method: set right vectors as random vectors, left vectors are entities @@ -889,7 +902,7 @@ class TestUtilityBase(TestcaseBase): partitions = collection_w.partitions vectors_r = cf.gen_vectors(nb // 2, default_dim) op_r = {"float_vectors": vectors_r} - params = {"metric": metric, "sqrt": sqrt} + params = {metric_field: metric, "sqrt": sqrt} for i in range(len(partitions)): vectors_l = vectors[i].loc[:, default_field_name] op_l = {"ids": insert_ids[middle:], "collection": collection_w.name, -- GitLab