admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: purge small files right after wp segment compaction (#44473)

Browse Source 
#43638 
improve wp log output
[wp#43](https://github.com/zilliztech/woodpecker/issues/43)
intro purge small files right after segment compaction
[wp#47](https://github.com/zilliztech/woodpecker/issues/47)
The rootpath configured by milvus is uniformly used as the base for wp
local fs storage.
update to v0.1.5

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
This commit is contained in:
tinswzy 2025-09-21 16:32:01 +08:00 committed by GitHub
parent d3784c6515
commit c7f21d5a06
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/milvus.yaml
View File

@ -212,7 +212,7 @@ woodpecker:
maxFetchThreads: 32 # Maximum number of threads to fetch data.
storage:
type: minio # The Type of the storage provider. Valid values: [minio, local]
rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider.
rootPath: default # The root path of the storage provider. If set to 'default', uses localStorage.path as base directory and creates a woodpecker subdirectory. Otherwise, specifies a custom woodpecker data storage directory.
# Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services.
pulsar:

2
go.mod
View File

@ -259,7 +259,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.1.4 // indirect
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1142,8 +1142,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.1.4 h1:X9DTl3xlad99VZcTHCw1AWQiT2pBBHnIl+SueZgBYoM=
github.com/zilliztech/woodpecker v0.1.4/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk=
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd h1:F23qw9L2g0Fczgtl274TTymz+cL5KNf+60525brrNnE=
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -40,7 +40,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.1.4
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -881,8 +881,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.1.4 h1:X9DTl3xlad99VZcTHCw1AWQiT2pBBHnIl+SueZgBYoM=
github.com/zilliztech/woodpecker v0.1.4/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk=
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd h1:F23qw9L2g0Fczgtl274TTymz+cL5KNf+60525brrNnE=
github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

10
pkg/streaming/walimpls/impls/wp/builder.go
View File

@ -109,7 +109,15 @@ func (b *builderImpl) setCustomWpConfig(wpConfig *config.Configuration, cfg *par
wpConfig.Woodpecker.Logstore.SegmentReadPolicy.MaxFetchThreads = cfg.ReaderMaxFetchThreads.GetAsInt()
// storage
wpConfig.Woodpecker.Storage.Type = cfg.StorageType.GetValue()
wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue()
// Set RootPath based on configuration
if cfg.RootPath.GetValue() == "default" {
// Use LocalStorage.Path as prefix with "wp" subdirectory for default
wpConfig.Woodpecker.Storage.RootPath = fmt.Sprintf("%s/wp", paramtable.Get().LocalStorageCfg.Path.GetValue())
} else {
// Use custom directory as-is
wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue()
}
// set bucketName
wpConfig.Minio.BucketName = paramtable.Get().MinioCfg.BucketName.GetValue()

4
pkg/util/paramtable/service_param.go
View File

@ -896,8 +896,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.RootPath = ParamItem{
Key: "woodpecker.storage.rootPath",
Version: "2.6.0",
DefaultValue: "/var/lib/milvus/woodpecker",
Doc: "The root path of the storage provider.",
DefaultValue: "default",
Doc: "The root path of the storage provider. If set to 'default', uses localStorage.path as base directory and creates a woodpecker subdirectory. Otherwise, specifies a custom woodpecker data storage directory.",
Export: true,
}
p.RootPath.Init(base.mgr)

2
pkg/util/paramtable/service_param_test.go
View File

@ -123,7 +123,7 @@ func TestServiceParam(t *testing.T) {
assert.Equal(t, wpCfg.ReaderMaxFetchThreads.GetAsInt(), 32)
assert.Equal(t, wpCfg.StorageType.GetValue(), "minio")
assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker")
assert.Equal(t, wpCfg.RootPath.GetValue(), "default")
})
t.Run("test pulsarConfig", func(t *testing.T) {