From c7f21d5a06ae2fd442239560291f4eced0ed391f Mon Sep 17 00:00:00 2001 From: tinswzy Date: Sun, 21 Sep 2025 16:32:01 +0800 Subject: [PATCH] enhance: purge small files right after wp segment compaction (#44473) #43638 improve wp log output [wp#43](https://github.com/zilliztech/woodpecker/issues/43) intro purge small files right after segment compaction [wp#47](https://github.com/zilliztech/woodpecker/issues/47) The rootpath configured by milvus is uniformly used as the base for wp local fs storage. update to v0.1.5 Signed-off-by: tinswzy --- configs/milvus.yaml | 2 +- go.mod | 2 +- go.sum | 4 ++-- pkg/go.mod | 2 +- pkg/go.sum | 4 ++-- pkg/streaming/walimpls/impls/wp/builder.go | 10 +++++++++- pkg/util/paramtable/service_param.go | 4 ++-- pkg/util/paramtable/service_param_test.go | 2 +- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/configs/milvus.yaml b/configs/milvus.yaml index e03615195d..d3cd0f7add 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -212,7 +212,7 @@ woodpecker: maxFetchThreads: 32 # Maximum number of threads to fetch data. storage: type: minio # The Type of the storage provider. Valid values: [minio, local] - rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider. + rootPath: default # The root path of the storage provider. If set to 'default', uses localStorage.path as base directory and creates a woodpecker subdirectory. Otherwise, specifies a custom woodpecker data storage directory. # Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services. pulsar: diff --git a/go.mod b/go.mod index 0c4243d97a..bc5f8c1207 100644 --- a/go.mod +++ b/go.mod @@ -259,7 +259,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - github.com/zilliztech/woodpecker v0.1.4 // indirect + github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd // indirect go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/v2 v2.305.5 // indirect diff --git a/go.sum b/go.sum index c30dd0f88f..7365902c09 100644 --- a/go.sum +++ b/go.sum @@ -1142,8 +1142,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -github.com/zilliztech/woodpecker v0.1.4 h1:X9DTl3xlad99VZcTHCw1AWQiT2pBBHnIl+SueZgBYoM= -github.com/zilliztech/woodpecker v0.1.4/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk= +github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd h1:F23qw9L2g0Fczgtl274TTymz+cL5KNf+60525brrNnE= +github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= diff --git a/pkg/go.mod b/pkg/go.mod index dd8d30bf33..7af4f98ac8 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -40,7 +40,7 @@ require ( github.com/tikv/client-go/v2 v2.0.4 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/x448/float16 v0.8.4 - github.com/zilliztech/woodpecker v0.1.4 + github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd go.etcd.io/etcd/api/v3 v3.5.5 go.etcd.io/etcd/client/v3 v3.5.5 go.etcd.io/etcd/server/v3 v3.5.5 diff --git a/pkg/go.sum b/pkg/go.sum index 92c100e735..12b9b9871a 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -881,8 +881,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zilliztech/woodpecker v0.1.4 h1:X9DTl3xlad99VZcTHCw1AWQiT2pBBHnIl+SueZgBYoM= -github.com/zilliztech/woodpecker v0.1.4/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk= +github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd h1:F23qw9L2g0Fczgtl274TTymz+cL5KNf+60525brrNnE= +github.com/zilliztech/woodpecker v0.1.5-0.20250919073140-d96966813dbd/go.mod h1:coEnEv3zqCUi6QFyhi7g+3FHkHb3D6gD6oNRxMjXnNk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= diff --git a/pkg/streaming/walimpls/impls/wp/builder.go b/pkg/streaming/walimpls/impls/wp/builder.go index f9d5209542..5bbf1962b5 100644 --- a/pkg/streaming/walimpls/impls/wp/builder.go +++ b/pkg/streaming/walimpls/impls/wp/builder.go @@ -109,7 +109,15 @@ func (b *builderImpl) setCustomWpConfig(wpConfig *config.Configuration, cfg *par wpConfig.Woodpecker.Logstore.SegmentReadPolicy.MaxFetchThreads = cfg.ReaderMaxFetchThreads.GetAsInt() // storage wpConfig.Woodpecker.Storage.Type = cfg.StorageType.GetValue() - wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue() + + // Set RootPath based on configuration + if cfg.RootPath.GetValue() == "default" { + // Use LocalStorage.Path as prefix with "wp" subdirectory for default + wpConfig.Woodpecker.Storage.RootPath = fmt.Sprintf("%s/wp", paramtable.Get().LocalStorageCfg.Path.GetValue()) + } else { + // Use custom directory as-is + wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue() + } // set bucketName wpConfig.Minio.BucketName = paramtable.Get().MinioCfg.BucketName.GetValue() diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index 076fa41915..d49afbab13 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -896,8 +896,8 @@ func (p *WoodpeckerConfig) Init(base *BaseTable) { p.RootPath = ParamItem{ Key: "woodpecker.storage.rootPath", Version: "2.6.0", - DefaultValue: "/var/lib/milvus/woodpecker", - Doc: "The root path of the storage provider.", + DefaultValue: "default", + Doc: "The root path of the storage provider. If set to 'default', uses localStorage.path as base directory and creates a woodpecker subdirectory. Otherwise, specifies a custom woodpecker data storage directory.", Export: true, } p.RootPath.Init(base.mgr) diff --git a/pkg/util/paramtable/service_param_test.go b/pkg/util/paramtable/service_param_test.go index cad23a1a53..bc4374308f 100644 --- a/pkg/util/paramtable/service_param_test.go +++ b/pkg/util/paramtable/service_param_test.go @@ -123,7 +123,7 @@ func TestServiceParam(t *testing.T) { assert.Equal(t, wpCfg.ReaderMaxFetchThreads.GetAsInt(), 32) assert.Equal(t, wpCfg.StorageType.GetValue(), "minio") - assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker") + assert.Equal(t, wpCfg.RootPath.GetValue(), "default") }) t.Run("test pulsarConfig", func(t *testing.T) {