From 754ca58469df3d3fd571a37791686c603f75bfe2 Mon Sep 17 00:00:00 2001 From: tinswzy Date: Fri, 13 Jun 2025 20:33:41 +0800 Subject: [PATCH] enhance: improve WP parameters according to performance testing (#42666) --- cmd/tools/config/generate.go | 11 ++++++----- configs/milvus.yaml | 19 ++++++++++--------- go.mod | 2 +- go.sum | 4 ++-- pkg/go.mod | 2 +- pkg/go.sum | 4 ++-- pkg/util/paramtable/service_param.go | 12 ++++++------ pkg/util/paramtable/service_param_test.go | 8 ++++---- 8 files changed, 32 insertions(+), 30 deletions(-) diff --git a/cmd/tools/config/generate.go b/cmd/tools/config/generate.go index 52ac7e5231..3f0a038a9a 100644 --- a/cmd/tools/config/generate.go +++ b/cmd/tools/config/generate.go @@ -233,11 +233,12 @@ func WriteYaml(w io.Writer) { { name: "mq", header: ` -# Milvus supports four MQ: rocksmq(based on RockDB), Pulsar and Kafka. -# You can change your mq by setting mq.type field. -# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file. -# 1. standalone(local) mode: rocksmq(default) > Pulsar > Kafka -# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported in cluster mode)`, +# Milvus supports four message queues (MQ): rocksmq (based on RocksDB), Pulsar, Kafka, and Woodpecker. +# You can change the MQ by setting the mq.type field. +# If the mq.type field is not set, the following priority is used when multiple MQs are configured in this file: +# 1. standalone (local) mode: rocksmq (default) > Pulsar > Kafka > Woodpecker +# 2. cluster mode: Pulsar (default) > Kafka (rocksmq is unsupported in cluster mode) > Woodpecker +# Note: These MQ priorities are compatible with existing instances. For new instances, it is recommended to explicitly use Woodpecker to achieve better performance, operational simplicity, and cost efficiency.`, }, { name: "woodpecker", diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 46bde5d841..a05cf640a5 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -158,11 +158,12 @@ minio: # 0 means using oss client by default, decrease these configration if ListObjects timeout listObjectsMaxKeys: 0 -# Milvus supports four MQ: rocksmq(based on RockDB), Pulsar and Kafka. -# You can change your mq by setting mq.type field. -# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file. -# 1. standalone(local) mode: rocksmq(default) > Pulsar > Kafka -# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported in cluster mode) +# Milvus supports four message queues (MQ): rocksmq (based on RocksDB), Pulsar, Kafka, and Woodpecker. +# You can change the MQ by setting the mq.type field. +# If the mq.type field is not set, the following priority is used when multiple MQs are configured in this file: +# 1. standalone (local) mode: rocksmq (default) > Pulsar > Kafka > Woodpecker +# 2. cluster mode: Pulsar (default) > Kafka (rocksmq is unsupported in cluster mode) > Woodpecker +# Note: These MQ priorities are compatible with existing instances. For new instances, it is recommended to explicitly use Woodpecker to achieve better performance, operational simplicity, and cost efficiency. mq: # Default value: "default" # Valid values: [default, pulsar, kafka, rocksmq, woodpecker] @@ -187,7 +188,7 @@ woodpecker: queueSize: 10000 # The size of the queue for pending messages to be sent of each log. maxRetries: 3 # Maximum number of retries for segment append operations. segmentRollingPolicy: - maxSize: 128M # Maximum entries count of a segment, default is 128M + maxSize: 2GB # Maximum size of a segment, default is 2GB maxInterval: 10m # Maximum interval between two segments, default is 10 minutes. auditor: maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds. @@ -195,12 +196,12 @@ woodpecker: segmentSyncPolicy: maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds. maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds. - maxBytes: 32M # Maximum size of write buffer in bytes. + maxBytes: 128M # Maximum size of write buffer in bytes. maxEntries: 10000 # Maximum entries number of write buffer. maxFlushRetries: 5 # Maximum size of write buffer in bytes. retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds. - maxFlushSize: 8M # Maximum size of a fragment in bytes to flush, default is 8M. - maxFlushThreads: 4 # Maximum number of threads to flush data + maxFlushSize: 4M # Maximum size of a fragment in bytes to flush, default is 4M. + maxFlushThreads: 32 # Maximum number of threads to flush data segmentCompactionPolicy: maxSize: 8M # The maximum size of the merged files, default is 8M. storage: diff --git a/go.mod b/go.mod index 8d9d4dd1de..0cdea61d83 100644 --- a/go.mod +++ b/go.mod @@ -243,7 +243,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 // indirect + github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/v2 v2.305.5 // indirect diff --git a/go.sum b/go.sum index bca54648e0..b843613ba2 100644 --- a/go.sum +++ b/go.sum @@ -1064,8 +1064,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg= -github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo= +github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 h1:tGSFnii0MD+hiaBltSg4QJH4UDm176+70V3yyF8/Rxg= +github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= diff --git a/pkg/go.mod b/pkg/go.mod index dc966637c3..d4047ec84d 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -39,7 +39,7 @@ require ( github.com/tikv/client-go/v2 v2.0.4 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/x448/float16 v0.8.4 - github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 + github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 go.etcd.io/etcd/api/v3 v3.5.5 go.etcd.io/etcd/client/v3 v3.5.5 go.etcd.io/etcd/server/v3 v3.5.5 diff --git a/pkg/go.sum b/pkg/go.sum index db0fc9585d..7ff49b7a7b 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -822,8 +822,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg= -github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo= +github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 h1:tGSFnii0MD+hiaBltSg4QJH4UDm176+70V3yyF8/Rxg= +github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index 76e7b90edb..ea18685025 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -729,8 +729,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) { p.SegmentRollingMaxSize = ParamItem{ Key: "woodpecker.client.segmentRollingPolicy.maxSize", Version: "2.6.0", - DefaultValue: "128M", - Doc: "Maximum entries count of a segment, default is 128M", + DefaultValue: "2GB", + Doc: "Maximum size of a segment, default is 2GB", Export: true, } p.SegmentRollingMaxSize.Init(base.mgr) @@ -783,7 +783,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) { p.SyncMaxBytes = ParamItem{ Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes", Version: "2.6.0", - DefaultValue: "32M", + DefaultValue: "128M", Doc: "Maximum size of write buffer in bytes.", Export: true, } @@ -801,8 +801,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) { p.FlushMaxSize = ParamItem{ Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize", Version: "2.6.0", - DefaultValue: "8M", - Doc: "Maximum size of a fragment in bytes to flush, default is 8M.", + DefaultValue: "4M", + Doc: "Maximum size of a fragment in bytes to flush, default is 4M.", Export: true, } p.FlushMaxSize.Init(base.mgr) @@ -819,7 +819,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) { p.FlushMaxThreads = ParamItem{ Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushThreads", Version: "2.6.0", - DefaultValue: "4", + DefaultValue: "32", Doc: "Maximum number of threads to flush data", Export: true, } diff --git a/pkg/util/paramtable/service_param_test.go b/pkg/util/paramtable/service_param_test.go index ed6c441ae4..daedbdda62 100644 --- a/pkg/util/paramtable/service_param_test.go +++ b/pkg/util/paramtable/service_param_test.go @@ -103,17 +103,17 @@ func TestServiceParam(t *testing.T) { assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000) assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3) - assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(128*1024*1024)) + assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(2*1024*1024*1024)) assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds(), float64(600)) assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10)) assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200)) assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10)) assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000) - assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(32*1024*1024)) + assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(128*1024*1024)) assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5) - assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(8*1024*1024)) - assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 4) + assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(4*1024*1024)) + assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 32) assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000)) assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(8*1024*1024))