admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refine Rocksdb option (#24394)

Browse Source 
Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd 2023-05-26 20:37:28 +08:00 committed by GitHub
parent 90f4425016
commit d11bcfefaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/core/thirdparty/rocksdb/CMakeLists.txt vendored
View File

@ -33,6 +33,7 @@ macro( build_rocksdb )
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
"-DROCKSDB_INSTALL_ON_WINDOWS=ON"
"-DFAIL_ON_WARNINGS=OFF"
"-DWITH_ZSTD=ON"
${FPIC_ARG}
#This is used to solve 'illegal instruction' problem in some machine
"-DPORTABLE=ON"

4
internal/core/thirdparty/rocksdb/rocksdb-win.pc.in vendored
View File

@ -5,6 +5,6 @@ Name: Rocksdb
Description: Rocksdb
Version: @ROCKSDB_VERSION@
Libs: -L${libdir} -lrocksdb
Libs.private: -lz -lbz2
Libs: -L${libdir} -lrocksdb -lzstd
Libs.private: -lz -lbz2
Cflags: -I${includedir}

2
internal/core/thirdparty/rocksdb/rocksdb.pc.in vendored
View File

@ -5,6 +5,6 @@ Name: Rocksdb
Description: Rocksdb
Version: @ROCKSDB_VERSION@
Libs: -L${libdir} -lrocksdb
Libs: -L${libdir} -lrocksdb -lzstd
Libs.private: -lz -lbz2
Cflags: -I${includedir}

10
internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go
View File

@ -154,9 +154,16 @@ func NewRocksMQ(params paramtable.BaseTable, name string, idAllocator allocator.
log.Info("Start rocksmq ", zap.Int("max proc", maxProcs),
zap.Int("parallism", parallelism), zap.Uint64("lru cache", rocksDBLRUCacheCapacity))
bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
bbto.SetBlockSize(64 << 10)
bbto.SetBlockCache(gorocksdb.NewLRUCache(rocksDBLRUCacheCapacity))
optsKV := gorocksdb.NewDefaultOptions()
// L0:No Compression
// L1,L2: ZSTD
optsKV.SetNumLevels(3)
optsKV.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7})
optsKV.SetBlockBasedTableFactory(bbto)
optsKV.SetTargetFileSizeMultiplier(2)
optsKV.SetCreateIfMissing(true)
// by default there are only 1 thread for flush compaction, which may block each other.
// increase to a reasonable thread numbers
@ -174,7 +181,10 @@ func NewRocksMQ(params paramtable.BaseTable, name string, idAllocator allocator.
// finish rocks mq store initialization, rocks mq store has to set the prefix extractor
optsStore := gorocksdb.NewDefaultOptions()
// share block cache with kv
optsKV.SetNumLevels(3)
optsStore.SetCompressionPerLevel([]gorocksdb.CompressionType{0, 7, 7})
optsStore.SetBlockBasedTableFactory(bbto)
optsStore.SetTargetFileSizeMultiplier(2)
optsStore.SetCreateIfMissing(true)
// by default there are only 1 thread for flush compaction, which may block each other.
// increase to a reasonable thread numbers