milvus/internal/datacoord/mock_handler.go
wei liu 975c91df16
feat: Add comprehensive snapshot functionality for collections (#44361)
issue: #44358

Implement complete snapshot management system including creation,
deletion, listing, description, and restoration capabilities across all
system components.

Key features:
- Create snapshots for entire collections
- Drop snapshots by name with proper cleanup
- List snapshots with collection filtering
- Describe snapshot details and metadata

Components added/modified:
- Client SDK with full snapshot API support and options
- DataCoord snapshot service with metadata management
- Proxy layer with task-based snapshot operations
- Protocol buffer definitions for snapshot RPCs
- Comprehensive unit tests with mockey framework
- Integration tests for end-to-end validation

Technical implementation:
- Snapshot metadata storage in etcd with proper indexing
- File-based snapshot data persistence in object storage
- Garbage collection integration for snapshot cleanup
- Error handling and validation across all operations
- Thread-safe operations with proper locking mechanisms

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
- Core invariant/assumption: snapshots are immutable point‑in‑time
captures identified by (collection, snapshot name/ID); etcd snapshot
metadata is authoritative for lifecycle (PENDING → COMMITTED → DELETING)
and per‑segment manifests live in object storage (Avro / StorageV2). GC
and restore logic must see snapshotRefIndex loaded
(snapshotMeta.IsRefIndexLoaded) before reclaiming or relying on
segment/index files.

- New capability added: full end‑to‑end snapshot subsystem — client SDK
APIs (Create/Drop/List/Describe/Restore + restore job queries),
DataCoord SnapshotWriter/Reader (Avro + StorageV2 manifests),
snapshotMeta in meta, SnapshotManager orchestration
(create/drop/describe/list/restore), copy‑segment restore
tasks/inspector/checker, proxy & RPC surface, GC integration, and
docs/tests — enabling point‑in‑time collection snapshots persisted to
object storage and restorations orchestrated across components.

- Logic removed/simplified and why: duplicated recursive
compaction/delta‑log traversal and ad‑hoc lookup code were consolidated
behind two focused APIs/owners (Handler.GetDeltaLogFromCompactTo for
delta traversal and SnapshotManager/SnapshotReader for snapshot I/O).
MixCoord/coordinator broker paths were converted to thin RPC proxies.
This eliminates multiple implementations of the same traversal/lookup,
reducing divergence and simplifying responsibility boundaries.

- Why this does NOT introduce data loss or regressions: snapshot
create/drop use explicit two‑phase semantics (PENDING → COMMIT/DELETING)
with SnapshotWriter writing manifests and metadata before commit; GC
uses snapshotRefIndex guards and
IsRefIndexLoaded/GetSnapshotBySegment/GetSnapshotByIndex checks to avoid
removing referenced files; restore flow pre‑allocates job IDs, validates
resources (partitions/indexes), performs rollback on failure
(rollbackRestoreSnapshot), and converts/updates segment/index metadata
only after successful copy tasks. Extensive unit and integration tests
exercise pending/deleting/GC/restore/error paths to ensure idempotence
and protection against premature deletion.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Wei Liu <wei.liu@zilliz.com>
2026-01-06 10:15:24 +08:00

542 lines
17 KiB
Go

// Code generated by mockery v2.53.3. DO NOT EDIT.
package datacoord
import (
context "context"
datapb "github.com/milvus-io/milvus/pkg/v2/proto/datapb"
mock "github.com/stretchr/testify/mock"
)
// NMockHandler is an autogenerated mock type for the Handler type
type NMockHandler struct {
mock.Mock
}
type NMockHandler_Expecter struct {
mock *mock.Mock
}
func (_m *NMockHandler) EXPECT() *NMockHandler_Expecter {
return &NMockHandler_Expecter{mock: &_m.Mock}
}
// CheckShouldDropChannel provides a mock function with given fields: ch
func (_m *NMockHandler) CheckShouldDropChannel(ch string) bool {
ret := _m.Called(ch)
if len(ret) == 0 {
panic("no return value specified for CheckShouldDropChannel")
}
var r0 bool
if rf, ok := ret.Get(0).(func(string) bool); ok {
r0 = rf(ch)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// NMockHandler_CheckShouldDropChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CheckShouldDropChannel'
type NMockHandler_CheckShouldDropChannel_Call struct {
*mock.Call
}
// CheckShouldDropChannel is a helper method to define mock.On call
// - ch string
func (_e *NMockHandler_Expecter) CheckShouldDropChannel(ch interface{}) *NMockHandler_CheckShouldDropChannel_Call {
return &NMockHandler_CheckShouldDropChannel_Call{Call: _e.mock.On("CheckShouldDropChannel", ch)}
}
func (_c *NMockHandler_CheckShouldDropChannel_Call) Run(run func(ch string)) *NMockHandler_CheckShouldDropChannel_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *NMockHandler_CheckShouldDropChannel_Call) Return(_a0 bool) *NMockHandler_CheckShouldDropChannel_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *NMockHandler_CheckShouldDropChannel_Call) RunAndReturn(run func(string) bool) *NMockHandler_CheckShouldDropChannel_Call {
_c.Call.Return(run)
return _c
}
// FinishDropChannel provides a mock function with given fields: ch, collectionID
func (_m *NMockHandler) FinishDropChannel(ch string, collectionID int64) error {
ret := _m.Called(ch, collectionID)
if len(ret) == 0 {
panic("no return value specified for FinishDropChannel")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, int64) error); ok {
r0 = rf(ch, collectionID)
} else {
r0 = ret.Error(0)
}
return r0
}
// NMockHandler_FinishDropChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FinishDropChannel'
type NMockHandler_FinishDropChannel_Call struct {
*mock.Call
}
// FinishDropChannel is a helper method to define mock.On call
// - ch string
// - collectionID int64
func (_e *NMockHandler_Expecter) FinishDropChannel(ch interface{}, collectionID interface{}) *NMockHandler_FinishDropChannel_Call {
return &NMockHandler_FinishDropChannel_Call{Call: _e.mock.On("FinishDropChannel", ch, collectionID)}
}
func (_c *NMockHandler_FinishDropChannel_Call) Run(run func(ch string, collectionID int64)) *NMockHandler_FinishDropChannel_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(int64))
})
return _c
}
func (_c *NMockHandler_FinishDropChannel_Call) Return(_a0 error) *NMockHandler_FinishDropChannel_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *NMockHandler_FinishDropChannel_Call) RunAndReturn(run func(string, int64) error) *NMockHandler_FinishDropChannel_Call {
_c.Call.Return(run)
return _c
}
// GenSnapshot provides a mock function with given fields: ctx, collectionID
func (_m *NMockHandler) GenSnapshot(ctx context.Context, collectionID int64) (*SnapshotData, error) {
ret := _m.Called(ctx, collectionID)
if len(ret) == 0 {
panic("no return value specified for GenSnapshot")
}
var r0 *SnapshotData
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64) (*SnapshotData, error)); ok {
return rf(ctx, collectionID)
}
if rf, ok := ret.Get(0).(func(context.Context, int64) *SnapshotData); ok {
r0 = rf(ctx, collectionID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*SnapshotData)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, collectionID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NMockHandler_GenSnapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GenSnapshot'
type NMockHandler_GenSnapshot_Call struct {
*mock.Call
}
// GenSnapshot is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
func (_e *NMockHandler_Expecter) GenSnapshot(ctx interface{}, collectionID interface{}) *NMockHandler_GenSnapshot_Call {
return &NMockHandler_GenSnapshot_Call{Call: _e.mock.On("GenSnapshot", ctx, collectionID)}
}
func (_c *NMockHandler_GenSnapshot_Call) Run(run func(ctx context.Context, collectionID int64)) *NMockHandler_GenSnapshot_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64))
})
return _c
}
func (_c *NMockHandler_GenSnapshot_Call) Return(_a0 *SnapshotData, _a1 error) *NMockHandler_GenSnapshot_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *NMockHandler_GenSnapshot_Call) RunAndReturn(run func(context.Context, int64) (*SnapshotData, error)) *NMockHandler_GenSnapshot_Call {
_c.Call.Return(run)
return _c
}
// GetCollection provides a mock function with given fields: ctx, collectionID
func (_m *NMockHandler) GetCollection(ctx context.Context, collectionID int64) (*collectionInfo, error) {
ret := _m.Called(ctx, collectionID)
if len(ret) == 0 {
panic("no return value specified for GetCollection")
}
var r0 *collectionInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64) (*collectionInfo, error)); ok {
return rf(ctx, collectionID)
}
if rf, ok := ret.Get(0).(func(context.Context, int64) *collectionInfo); ok {
r0 = rf(ctx, collectionID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*collectionInfo)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, collectionID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NMockHandler_GetCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCollection'
type NMockHandler_GetCollection_Call struct {
*mock.Call
}
// GetCollection is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
func (_e *NMockHandler_Expecter) GetCollection(ctx interface{}, collectionID interface{}) *NMockHandler_GetCollection_Call {
return &NMockHandler_GetCollection_Call{Call: _e.mock.On("GetCollection", ctx, collectionID)}
}
func (_c *NMockHandler_GetCollection_Call) Run(run func(ctx context.Context, collectionID int64)) *NMockHandler_GetCollection_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64))
})
return _c
}
func (_c *NMockHandler_GetCollection_Call) Return(_a0 *collectionInfo, _a1 error) *NMockHandler_GetCollection_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *NMockHandler_GetCollection_Call) RunAndReturn(run func(context.Context, int64) (*collectionInfo, error)) *NMockHandler_GetCollection_Call {
_c.Call.Return(run)
return _c
}
// GetCurrentSegmentsView provides a mock function with given fields: ctx, channel, partitionIDs
func (_m *NMockHandler) GetCurrentSegmentsView(ctx context.Context, channel RWChannel, partitionIDs ...int64) *SegmentsView {
_va := make([]interface{}, len(partitionIDs))
for _i := range partitionIDs {
_va[_i] = partitionIDs[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, channel)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for GetCurrentSegmentsView")
}
var r0 *SegmentsView
if rf, ok := ret.Get(0).(func(context.Context, RWChannel, ...int64) *SegmentsView); ok {
r0 = rf(ctx, channel, partitionIDs...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*SegmentsView)
}
}
return r0
}
// NMockHandler_GetCurrentSegmentsView_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCurrentSegmentsView'
type NMockHandler_GetCurrentSegmentsView_Call struct {
*mock.Call
}
// GetCurrentSegmentsView is a helper method to define mock.On call
// - ctx context.Context
// - channel RWChannel
// - partitionIDs ...int64
func (_e *NMockHandler_Expecter) GetCurrentSegmentsView(ctx interface{}, channel interface{}, partitionIDs ...interface{}) *NMockHandler_GetCurrentSegmentsView_Call {
return &NMockHandler_GetCurrentSegmentsView_Call{Call: _e.mock.On("GetCurrentSegmentsView",
append([]interface{}{ctx, channel}, partitionIDs...)...)}
}
func (_c *NMockHandler_GetCurrentSegmentsView_Call) Run(run func(ctx context.Context, channel RWChannel, partitionIDs ...int64)) *NMockHandler_GetCurrentSegmentsView_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]int64, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(int64)
}
}
run(args[0].(context.Context), args[1].(RWChannel), variadicArgs...)
})
return _c
}
func (_c *NMockHandler_GetCurrentSegmentsView_Call) Return(_a0 *SegmentsView) *NMockHandler_GetCurrentSegmentsView_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *NMockHandler_GetCurrentSegmentsView_Call) RunAndReturn(run func(context.Context, RWChannel, ...int64) *SegmentsView) *NMockHandler_GetCurrentSegmentsView_Call {
_c.Call.Return(run)
return _c
}
// GetDataVChanPositions provides a mock function with given fields: ch, partitionID
func (_m *NMockHandler) GetDataVChanPositions(ch RWChannel, partitionID int64) *datapb.VchannelInfo {
ret := _m.Called(ch, partitionID)
if len(ret) == 0 {
panic("no return value specified for GetDataVChanPositions")
}
var r0 *datapb.VchannelInfo
if rf, ok := ret.Get(0).(func(RWChannel, int64) *datapb.VchannelInfo); ok {
r0 = rf(ch, partitionID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*datapb.VchannelInfo)
}
}
return r0
}
// NMockHandler_GetDataVChanPositions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDataVChanPositions'
type NMockHandler_GetDataVChanPositions_Call struct {
*mock.Call
}
// GetDataVChanPositions is a helper method to define mock.On call
// - ch RWChannel
// - partitionID int64
func (_e *NMockHandler_Expecter) GetDataVChanPositions(ch interface{}, partitionID interface{}) *NMockHandler_GetDataVChanPositions_Call {
return &NMockHandler_GetDataVChanPositions_Call{Call: _e.mock.On("GetDataVChanPositions", ch, partitionID)}
}
func (_c *NMockHandler_GetDataVChanPositions_Call) Run(run func(ch RWChannel, partitionID int64)) *NMockHandler_GetDataVChanPositions_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(RWChannel), args[1].(int64))
})
return _c
}
func (_c *NMockHandler_GetDataVChanPositions_Call) Return(_a0 *datapb.VchannelInfo) *NMockHandler_GetDataVChanPositions_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *NMockHandler_GetDataVChanPositions_Call) RunAndReturn(run func(RWChannel, int64) *datapb.VchannelInfo) *NMockHandler_GetDataVChanPositions_Call {
_c.Call.Return(run)
return _c
}
// GetDeltaLogFromCompactTo provides a mock function with given fields: ctx, segmentID
func (_m *NMockHandler) GetDeltaLogFromCompactTo(ctx context.Context, segmentID int64) ([]*datapb.FieldBinlog, error) {
ret := _m.Called(ctx, segmentID)
if len(ret) == 0 {
panic("no return value specified for GetDeltaLogFromCompactTo")
}
var r0 []*datapb.FieldBinlog
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64) ([]*datapb.FieldBinlog, error)); ok {
return rf(ctx, segmentID)
}
if rf, ok := ret.Get(0).(func(context.Context, int64) []*datapb.FieldBinlog); ok {
r0 = rf(ctx, segmentID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*datapb.FieldBinlog)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, segmentID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NMockHandler_GetDeltaLogFromCompactTo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDeltaLogFromCompactTo'
type NMockHandler_GetDeltaLogFromCompactTo_Call struct {
*mock.Call
}
// GetDeltaLogFromCompactTo is a helper method to define mock.On call
// - ctx context.Context
// - segmentID int64
func (_e *NMockHandler_Expecter) GetDeltaLogFromCompactTo(ctx interface{}, segmentID interface{}) *NMockHandler_GetDeltaLogFromCompactTo_Call {
return &NMockHandler_GetDeltaLogFromCompactTo_Call{Call: _e.mock.On("GetDeltaLogFromCompactTo", ctx, segmentID)}
}
func (_c *NMockHandler_GetDeltaLogFromCompactTo_Call) Run(run func(ctx context.Context, segmentID int64)) *NMockHandler_GetDeltaLogFromCompactTo_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64))
})
return _c
}
func (_c *NMockHandler_GetDeltaLogFromCompactTo_Call) Return(_a0 []*datapb.FieldBinlog, _a1 error) *NMockHandler_GetDeltaLogFromCompactTo_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *NMockHandler_GetDeltaLogFromCompactTo_Call) RunAndReturn(run func(context.Context, int64) ([]*datapb.FieldBinlog, error)) *NMockHandler_GetDeltaLogFromCompactTo_Call {
_c.Call.Return(run)
return _c
}
// GetQueryVChanPositions provides a mock function with given fields: ch, partitionIDs
func (_m *NMockHandler) GetQueryVChanPositions(ch RWChannel, partitionIDs ...int64) *datapb.VchannelInfo {
_va := make([]interface{}, len(partitionIDs))
for _i := range partitionIDs {
_va[_i] = partitionIDs[_i]
}
var _ca []interface{}
_ca = append(_ca, ch)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for GetQueryVChanPositions")
}
var r0 *datapb.VchannelInfo
if rf, ok := ret.Get(0).(func(RWChannel, ...int64) *datapb.VchannelInfo); ok {
r0 = rf(ch, partitionIDs...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*datapb.VchannelInfo)
}
}
return r0
}
// NMockHandler_GetQueryVChanPositions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetQueryVChanPositions'
type NMockHandler_GetQueryVChanPositions_Call struct {
*mock.Call
}
// GetQueryVChanPositions is a helper method to define mock.On call
// - ch RWChannel
// - partitionIDs ...int64
func (_e *NMockHandler_Expecter) GetQueryVChanPositions(ch interface{}, partitionIDs ...interface{}) *NMockHandler_GetQueryVChanPositions_Call {
return &NMockHandler_GetQueryVChanPositions_Call{Call: _e.mock.On("GetQueryVChanPositions",
append([]interface{}{ch}, partitionIDs...)...)}
}
func (_c *NMockHandler_GetQueryVChanPositions_Call) Run(run func(ch RWChannel, partitionIDs ...int64)) *NMockHandler_GetQueryVChanPositions_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]int64, len(args)-1)
for i, a := range args[1:] {
if a != nil {
variadicArgs[i] = a.(int64)
}
}
run(args[0].(RWChannel), variadicArgs...)
})
return _c
}
func (_c *NMockHandler_GetQueryVChanPositions_Call) Return(_a0 *datapb.VchannelInfo) *NMockHandler_GetQueryVChanPositions_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *NMockHandler_GetQueryVChanPositions_Call) RunAndReturn(run func(RWChannel, ...int64) *datapb.VchannelInfo) *NMockHandler_GetQueryVChanPositions_Call {
_c.Call.Return(run)
return _c
}
// ListLoadedSegments provides a mock function with given fields: ctx
func (_m *NMockHandler) ListLoadedSegments(ctx context.Context) ([]int64, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for ListLoadedSegments")
}
var r0 []int64
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]int64, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []int64); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]int64)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NMockHandler_ListLoadedSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListLoadedSegments'
type NMockHandler_ListLoadedSegments_Call struct {
*mock.Call
}
// ListLoadedSegments is a helper method to define mock.On call
// - ctx context.Context
func (_e *NMockHandler_Expecter) ListLoadedSegments(ctx interface{}) *NMockHandler_ListLoadedSegments_Call {
return &NMockHandler_ListLoadedSegments_Call{Call: _e.mock.On("ListLoadedSegments", ctx)}
}
func (_c *NMockHandler_ListLoadedSegments_Call) Run(run func(ctx context.Context)) *NMockHandler_ListLoadedSegments_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *NMockHandler_ListLoadedSegments_Call) Return(_a0 []int64, _a1 error) *NMockHandler_ListLoadedSegments_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *NMockHandler_ListLoadedSegments_Call) RunAndReturn(run func(context.Context) ([]int64, error)) *NMockHandler_ListLoadedSegments_Call {
_c.Call.Return(run)
return _c
}
// NewNMockHandler creates a new instance of NMockHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewNMockHandler(t interface {
mock.TestingT
Cleanup(func())
}) *NMockHandler {
mock := &NMockHandler{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}