admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fence failure and remove list API usage (#43365)

Browse Source 
#43356  #43370 fence fail ； goroutine leaks
#43313 record too large

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
This commit is contained in:
tinswzy 2025-07-18 11:22:51 +08:00 committed by GitHub
parent d793def47c
commit 26f2de4bcf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
configs/milvus.yaml
View File

@ -188,7 +188,7 @@ woodpecker:
queueSize: 10000 # The size of the queue for pending messages to be sent of each log.
maxRetries: 3 # Maximum number of retries for segment append operations.
segmentRollingPolicy:
maxSize: 2GB # Maximum size of a segment, default is 2GB
maxSize: 256M # Maximum size of a segment.
maxInterval: 10m # Maximum interval between two segments, default is 10 minutes.
auditor:
maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds.
@ -196,14 +196,14 @@ woodpecker:
segmentSyncPolicy:
maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds.
maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds.
maxBytes: 128M # Maximum size of write buffer in bytes.
maxBytes: 256M # Maximum size of write buffer in bytes.
maxEntries: 10000 # Maximum entries number of write buffer.
maxFlushRetries: 5 # Maximum size of write buffer in bytes.
retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds.
maxFlushSize: 4M # Maximum size of a fragment in bytes to flush, default is 4M.
maxFlushSize: 2M # Maximum size of a fragment in bytes to flush.
maxFlushThreads: 32 # Maximum number of threads to flush data
segmentCompactionPolicy:
maxSize: 8M # The maximum size of the merged files, default is 8M.
maxSize: 2M # The maximum size of the merged files.
storage:
type: minio # The Type of the storage provider. Valid values: [minio, local]
rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider.

2
go.mod
View File

@ -247,7 +247,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 // indirect
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1079,8 +1079,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 h1:DLccWRhhx3V7HaRSujcBozpIsy1xOvs4jRgwBtihttw=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 h1:zzouo4FgdmibUjp7ilpdQVcEwljBFBFRc3TQKROrkwo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -822,8 +822,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 h1:DLccWRhhx3V7HaRSujcBozpIsy1xOvs4jRgwBtihttw=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 h1:zzouo4FgdmibUjp7ilpdQVcEwljBFBFRc3TQKROrkwo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

14
pkg/util/paramtable/service_param.go
View File

@ -729,8 +729,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SegmentRollingMaxSize = ParamItem{
Key: "woodpecker.client.segmentRollingPolicy.maxSize",
Version: "2.6.0",
DefaultValue: "2GB",
Doc: "Maximum size of a segment, default is 2GB",
DefaultValue: "256M",
Doc: "Maximum size of a segment.",
Export: true,
}
p.SegmentRollingMaxSize.Init(base.mgr)
@ -783,7 +783,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SyncMaxBytes = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes",
Version: "2.6.0",
DefaultValue: "128M",
DefaultValue: "256M",
Doc: "Maximum size of write buffer in bytes.",
Export: true,
}
@ -801,8 +801,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxSize = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize",
Version: "2.6.0",
DefaultValue: "4M",
Doc: "Maximum size of a fragment in bytes to flush, default is 4M.",
DefaultValue: "2M",
Doc: "Maximum size of a fragment in bytes to flush.",
Export: true,
}
p.FlushMaxSize.Init(base.mgr)
@ -828,8 +828,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.CompactionSize = ParamItem{
Key: "woodpecker.logstore.segmentCompactionPolicy.maxSize",
Version: "2.6.0",
DefaultValue: "8M",
Doc: "The maximum size of the merged files, default is 8M.",
DefaultValue: "2M",
Doc: "The maximum size of the merged files.",
Export: true,
}
p.CompactionSize.Init(base.mgr)

8
pkg/util/paramtable/service_param_test.go
View File

@ -103,19 +103,19 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000)
assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3)
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(2*1024*1024*1024))
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(256*1024*1024))
assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds(), float64(600))
assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10))
assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200))
assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10))
assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000)
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(128*1024*1024))
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(256*1024*1024))
assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5)
assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(4*1024*1024))
assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(2*1024*1024))
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 32)
assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000))
assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(8*1024*1024))
assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(2*1024*1024))
assert.Equal(t, wpCfg.StorageType.GetValue(), "minio")
assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker")