admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Adjust the default parameters of WP according to performance tests (#42598)

Browse Source 
#42595

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
This commit is contained in:
tinswzy 2025-06-10 16:30:35 +08:00 committed by GitHub
parent ed55b14484
commit 36a4b74fc0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 45 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
configs/milvus.yaml
View File

@ -192,19 +192,17 @@ woodpecker:
auditor:
maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds.
logstore:
logFileSyncPolicy:
segmentSyncPolicy:
maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds.
maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds.
maxBytes: 32M # Maximum size of write buffer in bytes.
maxEntries: 10000 # Maximum entries number of write buffer.
maxFlushRetries: 5 # Maximum size of write buffer in bytes.
retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds.
maxFlushSize: 8M # Maximum size of a fragment in bytes to flush, default is 8M.
maxFlushThreads: 4 # Maximum number of threads to flush data
logFileCompactionPolicy:
segmentCompactionPolicy:
maxSize: 8M # The maximum size of the merged files, default is 8M.
fragmentManager:
maxBytes: 128M # Maximum size of fragment cached data in bytes.
maxInterval: 1s # Maximum interval between two fragment evicts. default is 1 second
storage:
type: minio # The Type of the storage provider. Valid values: [minio, local]
rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider.

2
go.mod
View File

@ -243,7 +243,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1064,8 +1064,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958 h1:TYKnS3kpVOlE6oGI2rrq8yODBcnxUkzocRSPpySMjdo=
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -822,8 +822,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958 h1:TYKnS3kpVOlE6oGI2rrq8yODBcnxUkzocRSPpySMjdo=
github.com/zilliztech/woodpecker v0.0.0-20250607012818-0b658cd0c958/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254 h1:e0p+viwWPAHLhXqrlwSfqYnN23agGQtvyIW5tWc8kmg=
github.com/zilliztech/woodpecker v0.0.0-20250609083736-62920192a254/go.mod h1:MeyFx9vsAzxEysO2wzMC5d1QcSknbfVZOAhEmyxgQbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

19
pkg/streaming/walimpls/impls/wp/builder.go
View File

@ -97,16 +97,15 @@ func (b *builderImpl) setCustomWpConfig(wpConfig *config.Configuration, cfg *par
wpConfig.Woodpecker.Client.SegmentRollingPolicy.MaxSize = cfg.SegmentRollingMaxSize.GetAsSize()
wpConfig.Woodpecker.Client.SegmentRollingPolicy.MaxInterval = int(cfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds())
// logStore
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxInterval = int(cfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxEntries = cfg.SyncMaxEntries.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxBytes = cfg.SyncMaxBytes.GetAsSize()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushRetries = cfg.FlushMaxRetries.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushSize = cfg.FlushMaxSize.GetAsSize()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushThreads = cfg.FlushMaxThreads.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.RetryInterval = int(cfg.RetryInterval.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.LogFileCompactionPolicy.MaxBytes = cfg.CompactionSize.GetAsSize()
wpConfig.Woodpecker.Logstore.FragmentManager.MaxBytes = cfg.FragmentCachedMaxBytes.GetAsSize()
wpConfig.Woodpecker.Logstore.FragmentManager.MaxInterval = int(cfg.FragmentCachedInterval.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxInterval = int(cfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxIntervalForLocalStorage = int(cfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxEntries = cfg.SyncMaxEntries.GetAsInt()
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxBytes = cfg.SyncMaxBytes.GetAsSize()
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxFlushRetries = cfg.FlushMaxRetries.GetAsInt()
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxFlushSize = cfg.FlushMaxSize.GetAsSize()
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.MaxFlushThreads = cfg.FlushMaxThreads.GetAsInt()
wpConfig.Woodpecker.Logstore.SegmentSyncPolicy.RetryInterval = int(cfg.RetryInterval.GetAsDurationByParse().Milliseconds())
wpConfig.Woodpecker.Logstore.SegmentCompactionPolicy.MaxBytes = cfg.CompactionSize.GetAsSize()
// storage
wpConfig.Woodpecker.Storage.Type = cfg.StorageType.GetValue()
wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue()

46
pkg/util/paramtable/service_param.go
View File

@ -675,6 +675,7 @@ type WoodpeckerConfig struct {
// logstore
SyncMaxInterval ParamItem `refreshable:"true"`
SyncMaxIntervalForLocalStorage ParamItem `refreshable:"true"`
SyncMaxBytes ParamItem `refreshable:"true"`
SyncMaxEntries ParamItem `refreshable:"true"`
FlushMaxRetries ParamItem `refreshable:"true"`
@ -682,8 +683,6 @@ type WoodpeckerConfig struct {
FlushMaxSize ParamItem `refreshable:"true"`
FlushMaxThreads ParamItem `refreshable:"true"`
CompactionSize ParamItem `refreshable:"true"`
FragmentCachedMaxBytes ParamItem `refreshable:"true"`
FragmentCachedInterval ParamItem `refreshable:"true"`
// storage
StorageType ParamItem `refreshable:"false"`
@ -755,7 +754,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.AuditorMaxInterval.Init(base.mgr)
p.SyncMaxInterval = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxInterval",
Key: "woodpecker.logstore.segmentSyncPolicy.maxInterval",
Version: "2.6.0",
DefaultValue: "200ms",
Doc: "Maximum interval between two sync operations, default is 200 milliseconds.",
@ -763,8 +762,17 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
}
p.SyncMaxInterval.Init(base.mgr)
p.SyncMaxIntervalForLocalStorage = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxIntervalForLocalStorage",
Version: "2.6.0",
DefaultValue: "10ms",
Doc: "Maximum interval between two sync operations local storage backend, default is 10 milliseconds.",
Export: true,
}
p.SyncMaxIntervalForLocalStorage.Init(base.mgr)
p.SyncMaxEntries = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxEntries",
Key: "woodpecker.logstore.segmentSyncPolicy.maxEntries",
Version: "2.6.0",
DefaultValue: "10000",
Doc: "Maximum entries number of write buffer.",
@ -773,7 +781,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SyncMaxEntries.Init(base.mgr)
p.SyncMaxBytes = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxBytes",
Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes",
Version: "2.6.0",
DefaultValue: "32M",
Doc: "Maximum size of write buffer in bytes.",
@ -782,7 +790,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SyncMaxBytes.Init(base.mgr)
p.FlushMaxRetries = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushRetries",
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushRetries",
Version: "2.6.0",
DefaultValue: "5",
Doc: "Maximum size of write buffer in bytes.",
@ -791,7 +799,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxRetries.Init(base.mgr)
p.FlushMaxSize = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushSize",
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize",
Version: "2.6.0",
DefaultValue: "8M",
Doc: "Maximum size of a fragment in bytes to flush, default is 8M.",
@ -800,7 +808,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxSize.Init(base.mgr)
p.RetryInterval = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.retryInterval",
Key: "woodpecker.logstore.segmentSyncPolicy.retryInterval",
Version: "2.6.0",
DefaultValue: "1000ms",
Doc: "Maximum interval between two retries. default is 1000 milliseconds.",
@ -809,7 +817,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.RetryInterval.Init(base.mgr)
p.FlushMaxThreads = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushThreads",
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushThreads",
Version: "2.6.0",
DefaultValue: "4",
Doc: "Maximum number of threads to flush data",
@ -818,7 +826,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxThreads.Init(base.mgr)
p.CompactionSize = ParamItem{
Key: "woodpecker.logstore.logFileCompactionPolicy.maxSize",
Key: "woodpecker.logstore.segmentCompactionPolicy.maxSize",
Version: "2.6.0",
DefaultValue: "8M",
Doc: "The maximum size of the merged files, default is 8M.",
@ -826,24 +834,6 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
}
p.CompactionSize.Init(base.mgr)
p.FragmentCachedMaxBytes = ParamItem{
Key: "woodpecker.logstore.fragmentManager.maxBytes",
Version: "2.6.0",
DefaultValue: "128M",
Doc: "Maximum size of fragment cached data in bytes.",
Export: true,
}
p.FragmentCachedMaxBytes.Init(base.mgr)
p.FragmentCachedInterval = ParamItem{
Key: "woodpecker.logstore.fragmentManager.maxInterval",
Version: "2.6.0",
DefaultValue: "1s",
Doc: "Maximum interval between two fragment evicts. default is 1 second",
Export: true,
}
p.FragmentCachedInterval.Init(base.mgr)
p.StorageType = ParamItem{
Key: "woodpecker.storage.type",
Version: "2.6.0",

3
pkg/util/paramtable/service_param_test.go
View File

@ -108,6 +108,7 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10))
assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200))
assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10))
assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000)
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(32*1024*1024))
assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5)
@ -115,8 +116,6 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 4)
assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000))
assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(8*1024*1024))
assert.Equal(t, wpCfg.FragmentCachedMaxBytes.GetAsSize(), int64(128*1024*1024))
assert.Equal(t, wpCfg.FragmentCachedInterval.GetAsDurationByParse().Milliseconds(), int64(1000))
assert.Equal(t, wpCfg.StorageType.GetValue(), "minio")
assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker")