From c1e8406c442d23933d91403ff4c854fe664d1556 Mon Sep 17 00:00:00 2001 From: Gao Date: Sun, 30 Apr 2023 11:50:40 +0800 Subject: [PATCH] Override index type as AUTOINDEX when autoindex is enabled (#23744) Signed-off-by: chasingegg --- internal/proxy/task_index.go | 12 +-- internal/proxy/task_index_test.go | 98 ++++++++++++++++++++ tests/python_client/testcases/test_search.py | 31 +++++++ 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index ab36efc38..cb3c0131c 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -132,14 +132,10 @@ func (cit *createIndexTask) parseIndexParams() error { if isVecIndex { specifyIndexType, exist := indexParamsMap[common.IndexTypeKey] if Params.AutoIndexConfig.Enable.GetAsBool() { - if exist { - if specifyIndexType != AutoIndexName { - return fmt.Errorf("IndexType should be %s", AutoIndexName) - } - } - log.Debug("create index trigger AutoIndex", - zap.String("type", Params.AutoIndexConfig.AutoIndexTypeName.GetValue())) - // override params + log.Info("create index trigger AutoIndex", + zap.String("original type", specifyIndexType), + zap.String("final type", Params.AutoIndexConfig.AutoIndexTypeName.GetValue())) + // override params by autoindex for k, v := range Params.AutoIndexConfig.IndexParams.GetAsJSONMap() { indexParamsMap[k] = v } diff --git a/internal/proxy/task_index_test.go b/internal/proxy/task_index_test.go index 0fc0ff608..59d8979db 100644 --- a/internal/proxy/task_index_test.go +++ b/internal/proxy/task_index_test.go @@ -18,6 +18,7 @@ package proxy import ( "context" + "encoding/json" "os" "testing" @@ -358,4 +359,101 @@ func Test_parseIndexParams(t *testing.T) { }, }, cit.newTypeParams) }) + + cit2 := &createIndexTask{ + Condition: nil, + req: &milvuspb.CreateIndexRequest{ + Base: nil, + DbName: "", + CollectionName: "", + FieldName: "", + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: "index_type", + Value: "IVF_FLAT", + }, + { + Key: MetricTypeKey, + Value: "L2", + }, + { + Key: "params", + Value: "{\"nlist\": 100}", + }, + { + Key: DimKey, + Value: "128", + }, + }, + IndexName: "", + }, + ctx: nil, + rootCoord: nil, + queryCoord: nil, + result: nil, + isAutoIndex: false, + newIndexParams: nil, + newTypeParams: nil, + collectionID: 0, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + Description: "field no.1", + DataType: schemapb.DataType_FloatVector, + TypeParams: []*commonpb.KeyValuePair{ + { + Key: DimKey, + Value: "128", + }, + { + Key: MetricTypeKey, + Value: "L2", + }, + }}, + } + t.Run("parse index params 2", func(t *testing.T) { + Params.Save(Params.AutoIndexConfig.Enable.Key, "true") + indexParams := map[string]any{ + "index_type": "HNSW", + "M": 10, + "efConstruction": 100, + } + indexParamsStr, err := json.Marshal(indexParams) + assert.NoError(t, err) + Params.Save(Params.AutoIndexConfig.IndexParams.Key, string(indexParamsStr)) + err = cit2.parseIndexParams() + assert.NoError(t, err) + + assert.ElementsMatch(t, + []*commonpb.KeyValuePair{ + { + Key: "index_type", + Value: "HNSW", + }, + { + Key: MetricTypeKey, + Value: "L2", + }, + { + Key: "M", + Value: "10", + }, + { + Key: "efConstruction", + Value: "100", + }, + { + Key: "nlist", + Value: "100", + }, + }, cit2.newIndexParams) + assert.ElementsMatch(t, + []*commonpb.KeyValuePair{ + { + Key: DimKey, + Value: "128", + }, + }, cit2.newTypeParams) + }) } diff --git a/tests/python_client/testcases/test_search.py b/tests/python_client/testcases/test_search.py index 5e0012cf1..f8f1d188e 100644 --- a/tests/python_client/testcases/test_search.py +++ b/tests/python_client/testcases/test_search.py @@ -1744,6 +1744,37 @@ class TestCollectionSearch(TestcaseBase): "limit": default_limit, "_async": _async}) + @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.parametrize("M", [4, 64]) + @pytest.mark.parametrize("efConstruction", [8, 512]) + def test_search_HNSW_index_with_redundant_param(self, M, efConstruction, auto_id, _async): + """ + target: test search HNSW index with redundant param + method: connect milvus, create collection , insert, create index, load and search + expected: search successfully + """ + dim = M * 4 + self._connect() + collection_w, _, _, insert_ids, time_stamp = self.init_collection_general(prefix, True, + partition_num=1, + auto_id=auto_id, + dim=dim, is_index=False)[0:5] + HNSW_index_params = {"M": M, "efConstruction": efConstruction, "nlist": 100} # nlist is of no use + HNSW_index = {"index_type": "HNSW", "params": HNSW_index_params, "metric_type": "L2"} + collection_w.create_index("float_vector", HNSW_index) + collection_w.load() + search_param = {"metric_type": "L2", "params": {"ef": 32768, "nprobe": 10}} # nprobe is of no use + vectors = [[random.random() for _ in range(dim)] for _ in range(default_nq)] + collection_w.search(vectors[:default_nq], default_search_field, + search_param, default_limit, + default_search_exp, _async=_async, + travel_timestamp=0, + check_task=CheckTasks.check_search_results, + check_items={"nq": default_nq, + "ids": insert_ids, + "limit": default_limit, + "_async": _async}) + @pytest.mark.tags(CaseLabel.L1) @pytest.mark.parametrize("M", [4, 64]) @pytest.mark.parametrize("efConstruction", [8, 512]) -- GitLab