From 71cd7ba67a0b4026c02bae9705f337794177ce08 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Fri, 8 Apr 2022 20:29:33 +0800 Subject: [PATCH] Add configuration common.indexSliceSize (#16438) Signed-off-by: yudong.cai --- configs/milvus.yaml | 2 +- internal/core/src/config/ConfigKnowhere.cpp | 5 +++++ internal/core/src/config/ConfigKnowhere.h | 3 +++ internal/core/src/indexbuilder/init_c.cpp | 5 +++++ internal/core/src/indexbuilder/init_c.h | 3 +++ internal/core/src/segcore/segcore_init_c.cpp | 6 ++++++ internal/core/src/segcore/segcore_init_c.h | 3 +++ internal/core/thirdparty/knowhere/CMakeLists.txt | 4 ++-- internal/indexnode/indexnode.go | 4 ++++ internal/querynode/query_node.go | 4 ++++ internal/util/paramtable/component_param.go | 11 ++++++++++- internal/util/paramtable/component_param_test.go | 4 ++++ 12 files changed, 50 insertions(+), 4 deletions(-) diff --git a/configs/milvus.yaml b/configs/milvus.yaml index a51277da1..5d5e33e06 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -274,7 +274,7 @@ common: # Valid values: [auto, avx512, avx2, avx, sse4_2] # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building. simdType: auto + indexSliceSize: 4 # MB storage: vector: minio - diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp index 8fc42c3ef..ae6554871 100644 --- a/internal/core/src/config/ConfigKnowhere.cpp +++ b/internal/core/src/config/ConfigKnowhere.cpp @@ -62,4 +62,9 @@ KnowhereSetSimdType(const char* value) { } } +void +KnowhereSetIndexSliceSize(const int64_t size) { + knowhere::KnowhereConfig::SetIndexFileSliceSize(size); +} + } // namespace milvus::config diff --git a/internal/core/src/config/ConfigKnowhere.h b/internal/core/src/config/ConfigKnowhere.h index b180c860b..290fb1633 100644 --- a/internal/core/src/config/ConfigKnowhere.h +++ b/internal/core/src/config/ConfigKnowhere.h @@ -25,4 +25,7 @@ KnowhereInitImpl(); std::string KnowhereSetSimdType(const char*); +void +KnowhereSetIndexSliceSize(const int64_t size); + } // namespace milvus::config diff --git a/internal/core/src/indexbuilder/init_c.cpp b/internal/core/src/indexbuilder/init_c.cpp index cca543e60..e88c9bbee 100644 --- a/internal/core/src/indexbuilder/init_c.cpp +++ b/internal/core/src/indexbuilder/init_c.cpp @@ -27,3 +27,8 @@ IndexBuilderSetSimdType(const char* value) { ret[real_type.length()] = 0; return ret; } + +void +IndexBuilderSetIndexSliceSize(const int64_t value) { + milvus::config::KnowhereSetIndexSliceSize(value); +} diff --git a/internal/core/src/indexbuilder/init_c.h b/internal/core/src/indexbuilder/init_c.h index 5d15660a0..dbaa0c1cf 100644 --- a/internal/core/src/indexbuilder/init_c.h +++ b/internal/core/src/indexbuilder/init_c.h @@ -22,6 +22,9 @@ IndexBuilderInit(); char* IndexBuilderSetSimdType(const char*); +void +IndexBuilderSetIndexSliceSize(const int64_t); + #ifdef __cplusplus }; #endif diff --git a/internal/core/src/segcore/segcore_init_c.cpp b/internal/core/src/segcore/segcore_init_c.cpp index 659757a87..5b2bbd600 100644 --- a/internal/core/src/segcore/segcore_init_c.cpp +++ b/internal/core/src/segcore/segcore_init_c.cpp @@ -50,4 +50,10 @@ SegcoreSetSimdType(const char* value) { return ret; } +extern "C" void +SegcoreSetIndexSliceSize(const int64_t value) { + milvus::config::KnowhereSetIndexSliceSize(value); + LOG_SEGCORE_DEBUG_ << "set config index slice size: " << value; +} + } // namespace milvus::segcore diff --git a/internal/core/src/segcore/segcore_init_c.h b/internal/core/src/segcore/segcore_init_c.h index 0d77e1d3c..9a97552ad 100644 --- a/internal/core/src/segcore/segcore_init_c.h +++ b/internal/core/src/segcore/segcore_init_c.h @@ -25,6 +25,9 @@ SegcoreSetChunkRows(const int64_t); char* SegcoreSetSimdType(const char*); +void +SegcoreSetIndexSliceSize(const int64_t); + #ifdef __cplusplus } #endif diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index e039c0b95..bbd466de7 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -11,8 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. #------------------------------------------------------------------------------- -set( KNOWHERE_VERSION v1.1.2 ) -set( KNOWHERE_SOURCE_MD5 "7c6043bfaa073c6580a1c5b1e904c995" ) +set( KNOWHERE_VERSION v1.1.3 ) +set( KNOWHERE_SOURCE_MD5 "7497e91053608b354de7ef7f6c6ee54b" ) if ( DEFINED ENV{MILVUS_KNOWHERE_URL} ) set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" ) diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index e2ce31140..b8f004aa0 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -153,6 +153,10 @@ func (i *IndexNode) initKnowhere() { Params.CommonCfg.SimdType = C.GoString(cRealSimdType) C.free(unsafe.Pointer(cRealSimdType)) C.free(unsafe.Pointer(cSimdType)) + + // override segcore index slice size + cIndexSliceSize := C.int64_t(Params.CommonCfg.IndexSliceSize) + C.IndexBuilderSetIndexSliceSize(cIndexSliceSize) } func (i *IndexNode) initSession() error { diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index 5e84fb1d8..b91c1bbbf 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -185,6 +185,10 @@ func (node *QueryNode) InitSegcore() { Params.CommonCfg.SimdType = C.GoString(cRealSimdType) C.free(unsafe.Pointer(cRealSimdType)) C.free(unsafe.Pointer(cSimdType)) + + // override segcore index slice size + cIndexSliceSize := C.int64_t(Params.CommonCfg.IndexSliceSize) + C.SegcoreSetIndexSliceSize(cIndexSliceSize) } func (node *QueryNode) initServiceDiscovery() error { diff --git a/internal/util/paramtable/component_param.go b/internal/util/paramtable/component_param.go index 4e5edb58e..a30e5732a 100644 --- a/internal/util/paramtable/component_param.go +++ b/internal/util/paramtable/component_param.go @@ -28,6 +28,9 @@ import ( const ( // DefaultRetentionDuration defines the default duration for retention which is 5 days in seconds. DefaultRetentionDuration = 3600 * 24 * 5 + + // DefaultIndexSliceSize defines the default slice size of index file when serializing. + DefaultIndexSliceSize = 4 ) // ComponentParam is used to quickly and easily access all components' configurations. @@ -115,7 +118,8 @@ type commonConfig struct { DefaultIndexName string RetentionDuration int64 - SimdType string + SimdType string + IndexSliceSize int64 } func (p *commonConfig) init(base *BaseTable) { @@ -149,6 +153,7 @@ func (p *commonConfig) init(base *BaseTable) { p.initRetentionDuration() p.initSimdType() + p.initIndexSliceSize() } func (p *commonConfig) initClusterPrefix() { @@ -325,6 +330,10 @@ func (p *commonConfig) initSimdType() { p.SimdType = p.Base.LoadWithDefault2(keys, "auto") } +func (p *commonConfig) initIndexSliceSize() { + p.IndexSliceSize = p.Base.ParseInt64WithDefault("common.indexSliceSize", DefaultIndexSliceSize) +} + /////////////////////////////////////////////////////////////////////////////// // --- rootcoord --- type rootCoordConfig struct { diff --git a/internal/util/paramtable/component_param_test.go b/internal/util/paramtable/component_param_test.go index a6bc8a0e5..1a9a993bc 100644 --- a/internal/util/paramtable/component_param_test.go +++ b/internal/util/paramtable/component_param_test.go @@ -40,10 +40,14 @@ func TestComponentParam(t *testing.T) { t.Logf("default index name = %s", Params.DefaultIndexName) assert.Equal(t, Params.RetentionDuration, int64(DefaultRetentionDuration)) + t.Logf("default retention duration = %d", Params.RetentionDuration) assert.NotEqual(t, Params.SimdType, "") t.Logf("knowhere simd type = %s", Params.SimdType) + assert.Equal(t, Params.IndexSliceSize, int64(DefaultIndexSliceSize)) + t.Logf("knowhere index slice size = %d", Params.IndexSliceSize) + // -- proxy -- assert.Equal(t, Params.ProxySubName, "by-dev-proxy") t.Logf("ProxySubName: %s", Params.ProxySubName) -- GitLab