diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index 9762ab2bbc8551f2e177c5838afb54f5636675aa..a935354489c0a51905640b3c39601ccf71f7953a 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -50,6 +50,7 @@ class MilvusConan(ConanFile): generators = ("cmake", "cmake_find_package") default_options = { "rocksdb:shared": True, + "rocksdb:with_zstd": True, "arrow:parquet": True, "arrow:compute": True, "arrow:with_zstd": True, diff --git a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in index 60923fe9c967015ed24f99acd65b66b75745949f..5b423589e8edd386342806584a251e7ad25c804b 100644 --- a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in +++ b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in @@ -5,5 +5,5 @@ Description: Rocksdb Version: @ROCKSDB_VERSION@ Libs: -L${libdir} -lrocksdb -Libs.private: -lz -lbz2 │ +Libs.private: -lz -lbz2 Cflags: -I${includedir} diff --git a/internal/core/thirdparty/rocksdb/rocksdb.pc.in b/internal/core/thirdparty/rocksdb/rocksdb.pc.in index 4df5e7ffd36341b0ef38d68d78b05028b6978ad1..b3668975a64b8c726cce47bbceb64ebf05830954 100644 --- a/internal/core/thirdparty/rocksdb/rocksdb.pc.in +++ b/internal/core/thirdparty/rocksdb/rocksdb.pc.in @@ -4,6 +4,6 @@ Name: Rocksdb Description: Rocksdb Version: @ROCKSDB_VERSION@ -Libs: -L${libdir} -lrocksdb +Libs: -L${libdir} -lrocksdb Libs.private: -lz -lbz2 Cflags: -I${includedir} diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go b/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go index 2ed0dba7cf02a29e54333b8c499650d2522f3beb..383c6f086d0c742015ebf281e162ca97f22875f9 100644 --- a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go +++ b/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go @@ -159,9 +159,16 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error) log.Debug("Start rocksmq ", zap.Int("max proc", maxProcs), zap.Int("parallism", parallelism), zap.Uint64("lru cache", rocksDBLRUCacheCapacity)) bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockSize(64 << 10) bbto.SetBlockCache(gorocksdb.NewLRUCache(rocksDBLRUCacheCapacity)) + optsKV := gorocksdb.NewDefaultOptions() + // L0:No Compression + // L1,L2: ZSTD + optsKV.SetNumLevels(3) + optsKV.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7}) optsKV.SetBlockBasedTableFactory(bbto) + optsKV.SetTargetFileSizeMultiplier(2) optsKV.SetCreateIfMissing(true) // by default there are only 1 thread for flush compaction, which may block each other. // increase to a reasonable thread numbers @@ -179,7 +186,10 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error) // finish rocks mq store initialization, rocks mq store has to set the prefix extractor optsStore := gorocksdb.NewDefaultOptions() // share block cache with kv + optsKV.SetNumLevels(3) + optsStore.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7}) optsStore.SetBlockBasedTableFactory(bbto) + optsStore.SetTargetFileSizeMultiplier(2) optsStore.SetCreateIfMissing(true) // by default there are only 1 thread for flush compaction, which may block each other. // increase to a reasonable thread numbers