admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: improve WP parameters according to performance testing (#42666)

Browse Source
This commit is contained in:
tinswzy 2025-06-13 20:33:41 +08:00 committed by GitHub
parent ef8829c5bc
commit 754ca58469
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 32 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/tools/config/generate.go
View File

@ -233,11 +233,12 @@ func WriteYaml(w io.Writer) {
{
name: "mq",
header: `
# Milvus supports four MQ: rocksmq(based on RockDB), Pulsar and Kafka.
# You can change your mq by setting mq.type field.
# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file.
# 1. standalone(local) mode: rocksmq(default) > Pulsar > Kafka
# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported in cluster mode)`,
# Milvus supports four message queues (MQ): rocksmq (based on RocksDB), Pulsar, Kafka, and Woodpecker.
# You can change the MQ by setting the mq.type field.
# If the mq.type field is not set, the following priority is used when multiple MQs are configured in this file:
# 1. standalone (local) mode: rocksmq (default) > Pulsar > Kafka > Woodpecker
# 2. cluster mode: Pulsar (default) > Kafka (rocksmq is unsupported in cluster mode) > Woodpecker
# Note: These MQ priorities are compatible with existing instances. For new instances, it is recommended to explicitly use Woodpecker to achieve better performance, operational simplicity, and cost efficiency.`,
},
{
name: "woodpecker",

19
configs/milvus.yaml
View File

@ -158,11 +158,12 @@ minio:
# 0 means using oss client by default, decrease these configration if ListObjects timeout
listObjectsMaxKeys: 0
# Milvus supports four MQ: rocksmq(based on RockDB), Pulsar and Kafka.
# You can change your mq by setting mq.type field.
# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file.
# 1. standalone(local) mode: rocksmq(default) > Pulsar > Kafka
# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported in cluster mode)
# Milvus supports four message queues (MQ): rocksmq (based on RocksDB), Pulsar, Kafka, and Woodpecker.
# You can change the MQ by setting the mq.type field.
# If the mq.type field is not set, the following priority is used when multiple MQs are configured in this file:
# 1. standalone (local) mode: rocksmq (default) > Pulsar > Kafka > Woodpecker
# 2. cluster mode: Pulsar (default) > Kafka (rocksmq is unsupported in cluster mode) > Woodpecker
# Note: These MQ priorities are compatible with existing instances. For new instances, it is recommended to explicitly use Woodpecker to achieve better performance, operational simplicity, and cost efficiency.
mq:
# Default value: "default"
# Valid values: [default, pulsar, kafka, rocksmq, woodpecker]
@ -187,7 +188,7 @@ woodpecker:
queueSize: 10000 # The size of the queue for pending messages to be sent of each log.
maxRetries: 3 # Maximum number of retries for segment append operations.
segmentRollingPolicy:
maxSize: 128M # Maximum entries count of a segment, default is 128M
maxSize: 2GB # Maximum size of a segment, default is 2GB
maxInterval: 10m # Maximum interval between two segments, default is 10 minutes.
auditor:
maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds.
@ -195,12 +196,12 @@ woodpecker:
segmentSyncPolicy:
maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds.
maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds.
maxBytes: 32M # Maximum size of write buffer in bytes.
maxBytes: 128M # Maximum size of write buffer in bytes.
maxEntries: 10000 # Maximum entries number of write buffer.
maxFlushRetries: 5 # Maximum size of write buffer in bytes.
retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds.
maxFlushSize: 8M # Maximum size of a fragment in bytes to flush, default is 8M.
maxFlushThreads: 4 # Maximum number of threads to flush data
maxFlushSize: 4M # Maximum size of a fragment in bytes to flush, default is 4M.
maxFlushThreads: 32 # Maximum number of threads to flush data
segmentCompactionPolicy:
maxSize: 8M # The maximum size of the merged files, default is 8M.
storage:

2
go.mod
View File

@ -243,7 +243,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1064,8 +1064,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 h1:tGSFnii0MD+hiaBltSg4QJH4UDm176+70V3yyF8/Rxg=
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -822,8 +822,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39 h1:tGSFnii0MD+hiaBltSg4QJH4UDm176+70V3yyF8/Rxg=
github.com/zilliztech/woodpecker v0.0.0-20250613035007-382d1394eb39/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

12
pkg/util/paramtable/service_param.go
View File

@ -729,8 +729,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SegmentRollingMaxSize = ParamItem{
Key: "woodpecker.client.segmentRollingPolicy.maxSize",
Version: "2.6.0",
DefaultValue: "128M",
Doc: "Maximum entries count of a segment, default is 128M",
DefaultValue: "2GB",
Doc: "Maximum size of a segment, default is 2GB",
Export: true,
}
p.SegmentRollingMaxSize.Init(base.mgr)
@ -783,7 +783,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SyncMaxBytes = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes",
Version: "2.6.0",
DefaultValue: "32M",
DefaultValue: "128M",
Doc: "Maximum size of write buffer in bytes.",
Export: true,
}
@ -801,8 +801,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxSize = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize",
Version: "2.6.0",
DefaultValue: "8M",
Doc: "Maximum size of a fragment in bytes to flush, default is 8M.",
DefaultValue: "4M",
Doc: "Maximum size of a fragment in bytes to flush, default is 4M.",
Export: true,
}
p.FlushMaxSize.Init(base.mgr)
@ -819,7 +819,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxThreads = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushThreads",
Version: "2.6.0",
DefaultValue: "4",
DefaultValue: "32",
Doc: "Maximum number of threads to flush data",
Export: true,
}

8
pkg/util/paramtable/service_param_test.go
View File

@ -103,17 +103,17 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000)
assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3)
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(128*1024*1024))
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(2*1024*1024*1024))
assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds(), float64(600))
assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10))
assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200))
assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10))
assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000)
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(32*1024*1024))
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(128*1024*1024))
assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5)
assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(8*1024*1024))
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 4)
assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(4*1024*1024))
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 32)
assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000))
assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(8*1024*1024))