mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
issue: #44358 Implement complete snapshot management system including creation, deletion, listing, description, and restoration capabilities across all system components. Key features: - Create snapshots for entire collections - Drop snapshots by name with proper cleanup - List snapshots with collection filtering - Describe snapshot details and metadata Components added/modified: - Client SDK with full snapshot API support and options - DataCoord snapshot service with metadata management - Proxy layer with task-based snapshot operations - Protocol buffer definitions for snapshot RPCs - Comprehensive unit tests with mockey framework - Integration tests for end-to-end validation Technical implementation: - Snapshot metadata storage in etcd with proper indexing - File-based snapshot data persistence in object storage - Garbage collection integration for snapshot cleanup - Error handling and validation across all operations - Thread-safe operations with proper locking mechanisms <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant/assumption: snapshots are immutable point‑in‑time captures identified by (collection, snapshot name/ID); etcd snapshot metadata is authoritative for lifecycle (PENDING → COMMITTED → DELETING) and per‑segment manifests live in object storage (Avro / StorageV2). GC and restore logic must see snapshotRefIndex loaded (snapshotMeta.IsRefIndexLoaded) before reclaiming or relying on segment/index files. - New capability added: full end‑to‑end snapshot subsystem — client SDK APIs (Create/Drop/List/Describe/Restore + restore job queries), DataCoord SnapshotWriter/Reader (Avro + StorageV2 manifests), snapshotMeta in meta, SnapshotManager orchestration (create/drop/describe/list/restore), copy‑segment restore tasks/inspector/checker, proxy & RPC surface, GC integration, and docs/tests — enabling point‑in‑time collection snapshots persisted to object storage and restorations orchestrated across components. - Logic removed/simplified and why: duplicated recursive compaction/delta‑log traversal and ad‑hoc lookup code were consolidated behind two focused APIs/owners (Handler.GetDeltaLogFromCompactTo for delta traversal and SnapshotManager/SnapshotReader for snapshot I/O). MixCoord/coordinator broker paths were converted to thin RPC proxies. This eliminates multiple implementations of the same traversal/lookup, reducing divergence and simplifying responsibility boundaries. - Why this does NOT introduce data loss or regressions: snapshot create/drop use explicit two‑phase semantics (PENDING → COMMIT/DELETING) with SnapshotWriter writing manifests and metadata before commit; GC uses snapshotRefIndex guards and IsRefIndexLoaded/GetSnapshotBySegment/GetSnapshotByIndex checks to avoid removing referenced files; restore flow pre‑allocates job IDs, validates resources (partitions/indexes), performs rollback on failure (rollbackRestoreSnapshot), and converts/updates segment/index metadata only after successful copy tasks. Extensive unit and integration tests exercise pending/deleting/GC/restore/error paths to ensure idempotence and protection against premature deletion. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Wei Liu <wei.liu@zilliz.com>
155 lines
5.3 KiB
Go
155 lines
5.3 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package milvusclient
|
|
|
|
import (
|
|
"context"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
|
)
|
|
|
|
// CreateSnapshot creates a snapshot for the specified collection
|
|
func (c *Client) CreateSnapshot(ctx context.Context, opt CreateSnapshotOption, callOptions ...grpc.CallOption) error {
|
|
if opt == nil {
|
|
return merr.WrapErrParameterInvalid("CreateSnapshotOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
return c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.CreateSnapshot(ctx, req, callOptions...)
|
|
return merr.CheckRPCCall(resp, err)
|
|
})
|
|
}
|
|
|
|
// DropSnapshot drops a snapshot by name
|
|
func (c *Client) DropSnapshot(ctx context.Context, opt DropSnapshotOption, callOptions ...grpc.CallOption) error {
|
|
if opt == nil {
|
|
return merr.WrapErrParameterInvalid("DropSnapshotOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
return c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.DropSnapshot(ctx, req, callOptions...)
|
|
return merr.CheckRPCCall(resp, err)
|
|
})
|
|
}
|
|
|
|
// ListSnapshots lists all snapshots for the specified collection or all snapshots if no collection is specified
|
|
func (c *Client) ListSnapshots(ctx context.Context, opt ListSnapshotsOption, callOptions ...grpc.CallOption) ([]string, error) {
|
|
if opt == nil {
|
|
return nil, merr.WrapErrParameterInvalid("ListSnapshotsOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
var snapshots []string
|
|
|
|
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.ListSnapshots(ctx, req, callOptions...)
|
|
if err = merr.CheckRPCCall(resp, err); err != nil {
|
|
return err
|
|
}
|
|
snapshots = resp.GetSnapshots()
|
|
return nil
|
|
})
|
|
|
|
return snapshots, err
|
|
}
|
|
|
|
// DescribeSnapshot describes a snapshot by name
|
|
func (c *Client) DescribeSnapshot(ctx context.Context, opt DescribeSnapshotOption, callOptions ...grpc.CallOption) (*milvuspb.DescribeSnapshotResponse, error) {
|
|
if opt == nil {
|
|
return nil, merr.WrapErrParameterInvalid("DescribeSnapshotOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
var resp *milvuspb.DescribeSnapshotResponse
|
|
|
|
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
var err error
|
|
resp, err = milvusService.DescribeSnapshot(ctx, req, callOptions...)
|
|
return merr.CheckRPCCall(resp, err)
|
|
})
|
|
|
|
return resp, err
|
|
}
|
|
|
|
// RestoreSnapshot restores a snapshot to a target collection and returns the job ID for tracking
|
|
func (c *Client) RestoreSnapshot(ctx context.Context, opt RestoreSnapshotOption, callOptions ...grpc.CallOption) (int64, error) {
|
|
if opt == nil {
|
|
return 0, merr.WrapErrParameterInvalid("RestoreSnapshotOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
var jobID int64
|
|
|
|
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.RestoreSnapshot(ctx, req, callOptions...)
|
|
if err = merr.CheckRPCCall(resp, err); err != nil {
|
|
return err
|
|
}
|
|
jobID = resp.GetJobId()
|
|
return nil
|
|
})
|
|
|
|
return jobID, err
|
|
}
|
|
|
|
// GetRestoreSnapshotState gets the state of a restore snapshot job
|
|
func (c *Client) GetRestoreSnapshotState(ctx context.Context, opt GetRestoreSnapshotStateOption, callOptions ...grpc.CallOption) (*milvuspb.RestoreSnapshotInfo, error) {
|
|
if opt == nil {
|
|
return nil, merr.WrapErrParameterInvalid("GetRestoreSnapshotStateOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
var info *milvuspb.RestoreSnapshotInfo
|
|
|
|
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.GetRestoreSnapshotState(ctx, req, callOptions...)
|
|
if err = merr.CheckRPCCall(resp, err); err != nil {
|
|
return err
|
|
}
|
|
info = resp.GetInfo()
|
|
return nil
|
|
})
|
|
|
|
return info, err
|
|
}
|
|
|
|
// ListRestoreSnapshotJobs lists all restore snapshot jobs, optionally filtered by collection name
|
|
func (c *Client) ListRestoreSnapshotJobs(ctx context.Context, opt ListRestoreSnapshotJobsOption, callOptions ...grpc.CallOption) ([]*milvuspb.RestoreSnapshotInfo, error) {
|
|
if opt == nil {
|
|
return nil, merr.WrapErrParameterInvalid("ListRestoreSnapshotJobsOption", "nil", "option cannot be nil")
|
|
}
|
|
req := opt.Request()
|
|
|
|
var jobs []*milvuspb.RestoreSnapshotInfo
|
|
|
|
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
|
resp, err := milvusService.ListRestoreSnapshotJobs(ctx, req, callOptions...)
|
|
if err = merr.CheckRPCCall(resp, err); err != nil {
|
|
return err
|
|
}
|
|
jobs = resp.GetJobs()
|
|
return nil
|
|
})
|
|
|
|
return jobs, err
|
|
}
|