admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 09:08:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Add Truncate Interface to WALImpls for Log Retention Control (#41517)

Browse Source 
#41465 Add Truncate Interface to WALImpls for Proactive Log Retention
Management

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
This commit is contained in:
tinswzy 2025-04-29 14:36:49 +08:00 committed by GitHub
parent f3f8227cd0
commit b36ed03141
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 77 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -243,7 +243,7 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

4
go.sum
View File

@ -1065,8 +1065,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 h1:Te+TxCQVisH/ntsulwDlO77s3PuJhj//ok9xnaGupZo=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0 h1:6B7IUyTRarQVTvusRS0bs6aJn3tUTVTIVqPEOj5IQHM=
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

2
pkg/go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5

4
pkg/go.sum
View File

@ -823,8 +823,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 h1:Te+TxCQVisH/ntsulwDlO77s3PuJhj//ok9xnaGupZo=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0 h1:6B7IUyTRarQVTvusRS0bs6aJn3tUTVTIVqPEOj5IQHM=
github.com/zilliztech/woodpecker v0.0.0-20250427123625-654f0175eff0/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

47
pkg/mocks/streaming/mock_walimpls/mock_WALImpls.go
View File

@ -221,6 +221,53 @@ func (_c *MockWALImpls_Read_Call) RunAndReturn(run func(context.Context, walimpl
return _c
}
// Truncate provides a mock function with given fields: ctx, id
func (_m *MockWALImpls) Truncate(ctx context.Context, id message.MessageID) error {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for Truncate")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, message.MessageID) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockWALImpls_Truncate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Truncate'
type MockWALImpls_Truncate_Call struct {
*mock.Call
}
// Truncate is a helper method to define mock.On call
// - ctx context.Context
// - id message.MessageID
func (_e *MockWALImpls_Expecter) Truncate(ctx interface{}, id interface{}) *MockWALImpls_Truncate_Call {
return &MockWALImpls_Truncate_Call{Call: _e.mock.On("Truncate", ctx, id)}
}
func (_c *MockWALImpls_Truncate_Call) Run(run func(ctx context.Context, id message.MessageID)) *MockWALImpls_Truncate_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(message.MessageID))
})
return _c
}
func (_c *MockWALImpls_Truncate_Call) Return(_a0 error) *MockWALImpls_Truncate_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockWALImpls_Truncate_Call) RunAndReturn(run func(context.Context, message.MessageID) error) *MockWALImpls_Truncate_Call {
_c.Call.Return(run)
return _c
}
// WALName provides a mock function with no fields
func (_m *MockWALImpls) WALName() string {
ret := _m.Called()

4
pkg/streaming/walimpls/impls/kafka/wal.go
View File

@ -103,6 +103,10 @@ func (w *walImpl) Read(ctx context.Context, opt walimpls.ReadOption) (s walimpls
return newScanner(opt.Name, exclude, c), nil
}
func (w *walImpl) Truncate(ctx context.Context, id message.MessageID) error {
return nil
}
func (w *walImpl) Close() {
// The lifetime control of the producer is delegated to the wal adaptor.
// So we just make resource cleanup here.

4
pkg/streaming/walimpls/impls/pulsar/wal.go
View File

@ -76,6 +76,10 @@ func (w *walImpl) Read(ctx context.Context, opt walimpls.ReadOption) (s walimpls
return newScanner(opt.Name, reader), nil
}
func (w *walImpl) Truncate(ctx context.Context, id message.MessageID) error {
return nil
}
func (w *walImpl) Close() {
if w.p != nil {
w.p.Close() // close producer

4
pkg/streaming/walimpls/impls/rmq/wal.go
View File

@ -104,6 +104,10 @@ func (w *walImpl) Read(ctx context.Context, opt walimpls.ReadOption) (s walimpls
return newScanner(scannerName, exclude, consumer), nil
}
func (w *walImpl) Truncate(ctx context.Context, id message.MessageID) error {
return nil
}
// Close closes the wal.
func (w *walImpl) Close() {
if w.p != nil {

4
pkg/streaming/walimpls/impls/walimplstest/wal.go
View File

@ -56,5 +56,9 @@ func (w *walImpls) Read(ctx context.Context, opts walimpls.ReadOption) (walimpls
), nil
}
func (w *walImpls) Truncate(ctx context.Context, id message.MessageID) error {
return nil
}
func (w *walImpls) Close() {
}

6
pkg/streaming/walimpls/impls/wp/wal.go
View File

@ -62,13 +62,17 @@ func (w *walImpl) Read(ctx context.Context, opt walimpls.ReadOption) (walimpls.S
from.EntryId = id.logMsgId.EntryId + 1
}
reader, err := w.l.OpenLogReader(ctx, &from)
reader, err := w.l.OpenLogReader(ctx, &from, opt.Name)
if err != nil {
return nil, err
}
return newScanner(opt.Name, reader), nil
}
func (w *walImpl) Truncate(ctx context.Context, id message.MessageID) error {
return w.l.Truncate(ctx, id.(*wpID).logMsgId)
}
func (w *walImpl) Close() {
closeWriterErr := w.p.Close(context.Background())
if closeWriterErr != nil {

3
pkg/streaming/walimpls/wal.go
View File

@ -30,4 +30,7 @@ type WALImpls interface {
// Append writes a record to the log.
// Can be only called when the wal is in read-write mode.
Append(ctx context.Context, msg message.MutableMessage) (message.MessageID, error)
// Truncate truncates the wal to the given id (inclusive).
Truncate(ctx context.Context, id message.MessageID) error
}