admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fence failure and remove list API usage (#43365)

Browse Source 
#43356  #43370 fence fail ； goroutine leaks
#43313 record too large

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
This commit is contained in:
tinswzy 2025-07-18 11:22:51 +08:00 committed by GitHub
parent d793def47c
commit 26f2de4bcf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
configs/milvus.yaml
View File

@ -188,7 +188,7 @@ woodpecker:
queueSize: 10000 # The size of the queue for pending messages to be sent of each log. queueSize: 10000 # The size of the queue for pending messages to be sent of each log.
maxRetries: 3 # Maximum number of retries for segment append operations. maxRetries: 3 # Maximum number of retries for segment append operations.
segmentRollingPolicy: segmentRollingPolicy:
maxSize: 2GB # Maximum size of a segment, default is 2GB maxSize: 256M # Maximum size of a segment.
maxInterval: 10m # Maximum interval between two segments, default is 10 minutes. maxInterval: 10m # Maximum interval between two segments, default is 10 minutes.
auditor: auditor:
maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds. maxInterval: 10s # Maximum interval between two auditing operations, default is 10 seconds.
@ -196,14 +196,14 @@ woodpecker:
segmentSyncPolicy: segmentSyncPolicy:
maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds. maxInterval: 200ms # Maximum interval between two sync operations, default is 200 milliseconds.
maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds. maxIntervalForLocalStorage: 10ms # Maximum interval between two sync operations local storage backend, default is 10 milliseconds.
maxBytes: 128M # Maximum size of write buffer in bytes. maxBytes: 256M # Maximum size of write buffer in bytes.
maxEntries: 10000 # Maximum entries number of write buffer. maxEntries: 10000 # Maximum entries number of write buffer.
maxFlushRetries: 5 # Maximum size of write buffer in bytes. maxFlushRetries: 5 # Maximum size of write buffer in bytes.
retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds. retryInterval: 1000ms # Maximum interval between two retries. default is 1000 milliseconds.
maxFlushSize: 4M # Maximum size of a fragment in bytes to flush, default is 4M. maxFlushSize: 2M # Maximum size of a fragment in bytes to flush.
maxFlushThreads: 32 # Maximum number of threads to flush data maxFlushThreads: 32 # Maximum number of threads to flush data
segmentCompactionPolicy: segmentCompactionPolicy:
maxSize: 8M # The maximum size of the merged files, default is 8M. maxSize: 2M # The maximum size of the merged files.
storage: storage:
type: minio # The Type of the storage provider. Valid values: [minio, local] type: minio # The Type of the storage provider. Valid values: [minio, local]
rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider. rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider.

2
go.mod
View File

@ -247,7 +247,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 // indirect github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 // indirect
go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1079,8 +1079,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 h1:DLccWRhhx3V7HaRSujcBozpIsy1xOvs4jRgwBtihttw= github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 h1:zzouo4FgdmibUjp7ilpdQVcEwljBFBFRc3TQKROrkwo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo= github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4 github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4 github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7
go.etcd.io/etcd/api/v3 v3.5.5 go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5 go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5 go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -822,8 +822,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379 h1:DLccWRhhx3V7HaRSujcBozpIsy1xOvs4jRgwBtihttw= github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7 h1:zzouo4FgdmibUjp7ilpdQVcEwljBFBFRc3TQKROrkwo=
github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250715061301-59eac7b6e379/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo= github.com/zilliztech/woodpecker v0.1.0-rc2.0.20250717122411-9572cc8072a7/go.mod h1:vEEMNxlYFkNYvaCHcc83+HyVRIeZEjBTSMu+5dypKbo=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

14
pkg/util/paramtable/service_param.go
View File

@ -729,8 +729,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SegmentRollingMaxSize = ParamItem{ p.SegmentRollingMaxSize = ParamItem{
Key: "woodpecker.client.segmentRollingPolicy.maxSize", Key: "woodpecker.client.segmentRollingPolicy.maxSize",
Version: "2.6.0", Version: "2.6.0",
DefaultValue: "2GB", DefaultValue: "256M",
Doc: "Maximum size of a segment, default is 2GB", Doc: "Maximum size of a segment.",
Export: true, Export: true,
} }
p.SegmentRollingMaxSize.Init(base.mgr) p.SegmentRollingMaxSize.Init(base.mgr)
@ -783,7 +783,7 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.SyncMaxBytes = ParamItem{ p.SyncMaxBytes = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes", Key: "woodpecker.logstore.segmentSyncPolicy.maxBytes",
Version: "2.6.0", Version: "2.6.0",
DefaultValue: "128M", DefaultValue: "256M",
Doc: "Maximum size of write buffer in bytes.", Doc: "Maximum size of write buffer in bytes.",
Export: true, Export: true,
} }
@ -801,8 +801,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.FlushMaxSize = ParamItem{ p.FlushMaxSize = ParamItem{
Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize", Key: "woodpecker.logstore.segmentSyncPolicy.maxFlushSize",
Version: "2.6.0", Version: "2.6.0",
DefaultValue: "4M", DefaultValue: "2M",
Doc: "Maximum size of a fragment in bytes to flush, default is 4M.", Doc: "Maximum size of a fragment in bytes to flush.",
Export: true, Export: true,
} }
p.FlushMaxSize.Init(base.mgr) p.FlushMaxSize.Init(base.mgr)
@ -828,8 +828,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.CompactionSize = ParamItem{ p.CompactionSize = ParamItem{
Key: "woodpecker.logstore.segmentCompactionPolicy.maxSize", Key: "woodpecker.logstore.segmentCompactionPolicy.maxSize",
Version: "2.6.0", Version: "2.6.0",
DefaultValue: "8M", DefaultValue: "2M",
Doc: "The maximum size of the merged files, default is 8M.", Doc: "The maximum size of the merged files.",
Export: true, Export: true,
} }
p.CompactionSize.Init(base.mgr) p.CompactionSize.Init(base.mgr)

8
pkg/util/paramtable/service_param_test.go
View File

@ -103,19 +103,19 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000) assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000)
assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3) assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3)
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(2*1024*1024*1024)) assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsSize(), int64(256*1024*1024))
assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds(), float64(600)) assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsDurationByParse().Seconds(), float64(600))
assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10)) assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsDurationByParse().Seconds(), float64(10))
assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200)) assert.Equal(t, wpCfg.SyncMaxInterval.GetAsDurationByParse().Milliseconds(), int64(200))
assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10)) assert.Equal(t, wpCfg.SyncMaxIntervalForLocalStorage.GetAsDurationByParse().Milliseconds(), int64(10))
assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000) assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 10000)
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(128*1024*1024)) assert.Equal(t, wpCfg.SyncMaxBytes.GetAsSize(), int64(256*1024*1024))
assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5) assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5)
assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(4*1024*1024)) assert.Equal(t, wpCfg.FlushMaxSize.GetAsSize(), int64(2*1024*1024))
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 32) assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 32)
assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000)) assert.Equal(t, wpCfg.RetryInterval.GetAsDurationByParse().Milliseconds(), int64(1000))
assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(8*1024*1024)) assert.Equal(t, wpCfg.CompactionSize.GetAsSize(), int64(2*1024*1024))
assert.Equal(t, wpCfg.StorageType.GetValue(), "minio") assert.Equal(t, wpCfg.StorageType.GetValue(), "minio")
assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker") assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker")