mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
issue: #44358 Implement complete snapshot management system including creation, deletion, listing, description, and restoration capabilities across all system components. Key features: - Create snapshots for entire collections - Drop snapshots by name with proper cleanup - List snapshots with collection filtering - Describe snapshot details and metadata Components added/modified: - Client SDK with full snapshot API support and options - DataCoord snapshot service with metadata management - Proxy layer with task-based snapshot operations - Protocol buffer definitions for snapshot RPCs - Comprehensive unit tests with mockey framework - Integration tests for end-to-end validation Technical implementation: - Snapshot metadata storage in etcd with proper indexing - File-based snapshot data persistence in object storage - Garbage collection integration for snapshot cleanup - Error handling and validation across all operations - Thread-safe operations with proper locking mechanisms <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant/assumption: snapshots are immutable point‑in‑time captures identified by (collection, snapshot name/ID); etcd snapshot metadata is authoritative for lifecycle (PENDING → COMMITTED → DELETING) and per‑segment manifests live in object storage (Avro / StorageV2). GC and restore logic must see snapshotRefIndex loaded (snapshotMeta.IsRefIndexLoaded) before reclaiming or relying on segment/index files. - New capability added: full end‑to‑end snapshot subsystem — client SDK APIs (Create/Drop/List/Describe/Restore + restore job queries), DataCoord SnapshotWriter/Reader (Avro + StorageV2 manifests), snapshotMeta in meta, SnapshotManager orchestration (create/drop/describe/list/restore), copy‑segment restore tasks/inspector/checker, proxy & RPC surface, GC integration, and docs/tests — enabling point‑in‑time collection snapshots persisted to object storage and restorations orchestrated across components. - Logic removed/simplified and why: duplicated recursive compaction/delta‑log traversal and ad‑hoc lookup code were consolidated behind two focused APIs/owners (Handler.GetDeltaLogFromCompactTo for delta traversal and SnapshotManager/SnapshotReader for snapshot I/O). MixCoord/coordinator broker paths were converted to thin RPC proxies. This eliminates multiple implementations of the same traversal/lookup, reducing divergence and simplifying responsibility boundaries. - Why this does NOT introduce data loss or regressions: snapshot create/drop use explicit two‑phase semantics (PENDING → COMMIT/DELETING) with SnapshotWriter writing manifests and metadata before commit; GC uses snapshotRefIndex guards and IsRefIndexLoaded/GetSnapshotBySegment/GetSnapshotByIndex checks to avoid removing referenced files; restore flow pre‑allocates job IDs, validates resources (partitions/indexes), performs rollback on failure (rollbackRestoreSnapshot), and converts/updates segment/index metadata only after successful copy tasks. Extensive unit and integration tests exercise pending/deleting/GC/restore/error paths to ensure idempotence and protection against premature deletion. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Wei Liu <wei.liu@zilliz.com>
224 lines
6.2 KiB
Go
224 lines
6.2 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package milvusclient
|
|
|
|
import (
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
|
)
|
|
|
|
// CreateSnapshotOption interface for creating snapshot options
|
|
type CreateSnapshotOption interface {
|
|
Request() *milvuspb.CreateSnapshotRequest
|
|
}
|
|
|
|
type createSnapshotOption struct {
|
|
dbName string
|
|
collectionName string
|
|
name string
|
|
description string
|
|
}
|
|
|
|
func (opt *createSnapshotOption) Request() *milvuspb.CreateSnapshotRequest {
|
|
return &milvuspb.CreateSnapshotRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
Name: opt.name,
|
|
Description: opt.description,
|
|
DbName: opt.dbName,
|
|
CollectionName: opt.collectionName,
|
|
}
|
|
}
|
|
|
|
func (opt *createSnapshotOption) WithDescription(description string) *createSnapshotOption {
|
|
opt.description = description
|
|
return opt
|
|
}
|
|
|
|
func (opt *createSnapshotOption) WithDbName(dbName string) *createSnapshotOption {
|
|
opt.dbName = dbName
|
|
return opt
|
|
}
|
|
|
|
// NewCreateSnapshotOption creates a new CreateSnapshotOption
|
|
func NewCreateSnapshotOption(name string, collectionName string) *createSnapshotOption {
|
|
return &createSnapshotOption{
|
|
collectionName: collectionName,
|
|
name: name,
|
|
}
|
|
}
|
|
|
|
// DropSnapshotOption interface for dropping snapshot options
|
|
type DropSnapshotOption interface {
|
|
Request() *milvuspb.DropSnapshotRequest
|
|
}
|
|
|
|
type dropSnapshotOption struct {
|
|
name string
|
|
}
|
|
|
|
func (opt *dropSnapshotOption) Request() *milvuspb.DropSnapshotRequest {
|
|
return &milvuspb.DropSnapshotRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
Name: opt.name,
|
|
}
|
|
}
|
|
|
|
// NewDropSnapshotOption creates a new DropSnapshotOption
|
|
func NewDropSnapshotOption(name string) *dropSnapshotOption {
|
|
return &dropSnapshotOption{
|
|
name: name,
|
|
}
|
|
}
|
|
|
|
// ListSnapshotsOption interface for listing snapshots options
|
|
type ListSnapshotsOption interface {
|
|
Request() *milvuspb.ListSnapshotsRequest
|
|
}
|
|
|
|
type listSnapshotsOption struct {
|
|
dbName string
|
|
collectionName string
|
|
}
|
|
|
|
func (opt *listSnapshotsOption) Request() *milvuspb.ListSnapshotsRequest {
|
|
return &milvuspb.ListSnapshotsRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
DbName: opt.dbName,
|
|
CollectionName: opt.collectionName,
|
|
}
|
|
}
|
|
|
|
func (opt *listSnapshotsOption) WithCollectionName(collectionName string) *listSnapshotsOption {
|
|
opt.collectionName = collectionName
|
|
return opt
|
|
}
|
|
|
|
func (opt *listSnapshotsOption) WithDbName(dbName string) *listSnapshotsOption {
|
|
opt.dbName = dbName
|
|
return opt
|
|
}
|
|
|
|
// NewListSnapshotsOption creates a new ListSnapshotsOption
|
|
func NewListSnapshotsOption() *listSnapshotsOption {
|
|
return &listSnapshotsOption{}
|
|
}
|
|
|
|
// DescribeSnapshotOption interface for describing snapshot options
|
|
type DescribeSnapshotOption interface {
|
|
Request() *milvuspb.DescribeSnapshotRequest
|
|
}
|
|
|
|
type describeSnapshotOption struct {
|
|
name string
|
|
}
|
|
|
|
func (opt *describeSnapshotOption) Request() *milvuspb.DescribeSnapshotRequest {
|
|
return &milvuspb.DescribeSnapshotRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
Name: opt.name,
|
|
}
|
|
}
|
|
|
|
// NewDescribeSnapshotOption creates a new DescribeSnapshotOption
|
|
func NewDescribeSnapshotOption(name string) *describeSnapshotOption {
|
|
return &describeSnapshotOption{
|
|
name: name,
|
|
}
|
|
}
|
|
|
|
// RestoreSnapshotOption interface for restoring snapshot options
|
|
type RestoreSnapshotOption interface {
|
|
Request() *milvuspb.RestoreSnapshotRequest
|
|
}
|
|
|
|
type restoreSnapshotOption struct {
|
|
dbName string
|
|
name string
|
|
collectionName string
|
|
}
|
|
|
|
func (opt *restoreSnapshotOption) Request() *milvuspb.RestoreSnapshotRequest {
|
|
return &milvuspb.RestoreSnapshotRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
Name: opt.name,
|
|
DbName: opt.dbName,
|
|
CollectionName: opt.collectionName,
|
|
}
|
|
}
|
|
|
|
func (opt *restoreSnapshotOption) WithDbName(dbName string) *restoreSnapshotOption {
|
|
opt.dbName = dbName
|
|
return opt
|
|
}
|
|
|
|
// NewRestoreSnapshotOption creates a new RestoreSnapshotOption
|
|
func NewRestoreSnapshotOption(name string, collectionName string) *restoreSnapshotOption {
|
|
return &restoreSnapshotOption{
|
|
name: name,
|
|
collectionName: collectionName,
|
|
}
|
|
}
|
|
|
|
// GetRestoreSnapshotStateOption interface for getting restore snapshot state options
|
|
type GetRestoreSnapshotStateOption interface {
|
|
Request() *milvuspb.GetRestoreSnapshotStateRequest
|
|
}
|
|
|
|
type getRestoreSnapshotStateOption struct {
|
|
jobID int64
|
|
}
|
|
|
|
func (opt *getRestoreSnapshotStateOption) Request() *milvuspb.GetRestoreSnapshotStateRequest {
|
|
return &milvuspb.GetRestoreSnapshotStateRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
JobId: opt.jobID,
|
|
}
|
|
}
|
|
|
|
// NewGetRestoreSnapshotStateOption creates a new GetRestoreSnapshotStateOption
|
|
func NewGetRestoreSnapshotStateOption(jobID int64) *getRestoreSnapshotStateOption {
|
|
return &getRestoreSnapshotStateOption{
|
|
jobID: jobID,
|
|
}
|
|
}
|
|
|
|
// ListRestoreSnapshotJobsOption interface for listing restore snapshot jobs options
|
|
type ListRestoreSnapshotJobsOption interface {
|
|
Request() *milvuspb.ListRestoreSnapshotJobsRequest
|
|
}
|
|
|
|
type listRestoreSnapshotJobsOption struct {
|
|
collectionName string
|
|
}
|
|
|
|
func (opt *listRestoreSnapshotJobsOption) Request() *milvuspb.ListRestoreSnapshotJobsRequest {
|
|
return &milvuspb.ListRestoreSnapshotJobsRequest{
|
|
Base: &commonpb.MsgBase{},
|
|
CollectionName: opt.collectionName,
|
|
}
|
|
}
|
|
|
|
func (opt *listRestoreSnapshotJobsOption) WithCollectionName(collectionName string) *listRestoreSnapshotJobsOption {
|
|
opt.collectionName = collectionName
|
|
return opt
|
|
}
|
|
|
|
// NewListRestoreSnapshotJobsOption creates a new ListRestoreSnapshotJobsOption
|
|
func NewListRestoreSnapshotJobsOption() *listRestoreSnapshotJobsOption {
|
|
return &listRestoreSnapshotJobsOption{}
|
|
}
|