mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
issue: #44358 Implement complete snapshot management system including creation, deletion, listing, description, and restoration capabilities across all system components. Key features: - Create snapshots for entire collections - Drop snapshots by name with proper cleanup - List snapshots with collection filtering - Describe snapshot details and metadata Components added/modified: - Client SDK with full snapshot API support and options - DataCoord snapshot service with metadata management - Proxy layer with task-based snapshot operations - Protocol buffer definitions for snapshot RPCs - Comprehensive unit tests with mockey framework - Integration tests for end-to-end validation Technical implementation: - Snapshot metadata storage in etcd with proper indexing - File-based snapshot data persistence in object storage - Garbage collection integration for snapshot cleanup - Error handling and validation across all operations - Thread-safe operations with proper locking mechanisms <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant/assumption: snapshots are immutable point‑in‑time captures identified by (collection, snapshot name/ID); etcd snapshot metadata is authoritative for lifecycle (PENDING → COMMITTED → DELETING) and per‑segment manifests live in object storage (Avro / StorageV2). GC and restore logic must see snapshotRefIndex loaded (snapshotMeta.IsRefIndexLoaded) before reclaiming or relying on segment/index files. - New capability added: full end‑to‑end snapshot subsystem — client SDK APIs (Create/Drop/List/Describe/Restore + restore job queries), DataCoord SnapshotWriter/Reader (Avro + StorageV2 manifests), snapshotMeta in meta, SnapshotManager orchestration (create/drop/describe/list/restore), copy‑segment restore tasks/inspector/checker, proxy & RPC surface, GC integration, and docs/tests — enabling point‑in‑time collection snapshots persisted to object storage and restorations orchestrated across components. - Logic removed/simplified and why: duplicated recursive compaction/delta‑log traversal and ad‑hoc lookup code were consolidated behind two focused APIs/owners (Handler.GetDeltaLogFromCompactTo for delta traversal and SnapshotManager/SnapshotReader for snapshot I/O). MixCoord/coordinator broker paths were converted to thin RPC proxies. This eliminates multiple implementations of the same traversal/lookup, reducing divergence and simplifying responsibility boundaries. - Why this does NOT introduce data loss or regressions: snapshot create/drop use explicit two‑phase semantics (PENDING → COMMIT/DELETING) with SnapshotWriter writing manifests and metadata before commit; GC uses snapshotRefIndex guards and IsRefIndexLoaded/GetSnapshotBySegment/GetSnapshotByIndex checks to avoid removing referenced files; restore flow pre‑allocates job IDs, validates resources (partitions/indexes), performs rollback on failure (rollbackRestoreSnapshot), and converts/updates segment/index metadata only after successful copy tasks. Extensive unit and integration tests exercise pending/deleting/GC/restore/error paths to ensure idempotence and protection against premature deletion. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Wei Liu <wei.liu@zilliz.com>
166 lines
4.6 KiB
Go
166 lines
4.6 KiB
Go
package message
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/milvus-io/milvus/pkg/v2/proto/messagespb"
|
|
"github.com/milvus-io/milvus/pkg/v2/util"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
|
|
)
|
|
|
|
// NewResourceKeyFromProto creates a ResourceKey from proto.
|
|
func NewResourceKeyFromProto(proto *messagespb.ResourceKey) ResourceKey {
|
|
return ResourceKey{
|
|
Domain: proto.Domain,
|
|
Key: proto.Key,
|
|
Shared: proto.Shared,
|
|
}
|
|
}
|
|
|
|
// newProtoFromResourceKey creates a set of proto from ResourceKey.
|
|
func newProtoFromResourceKey(keys ...ResourceKey) []*messagespb.ResourceKey {
|
|
deduplicated := typeutil.NewSet(keys...)
|
|
protos := make([]*messagespb.ResourceKey, 0, len(keys))
|
|
for key := range deduplicated {
|
|
protos = append(protos, &messagespb.ResourceKey{
|
|
Domain: key.Domain,
|
|
Key: key.Key,
|
|
Shared: key.Shared,
|
|
})
|
|
}
|
|
return protos
|
|
}
|
|
|
|
type ResourceKey struct {
|
|
Domain messagespb.ResourceDomain
|
|
Key string
|
|
Shared bool
|
|
}
|
|
|
|
func (r ResourceKey) String() string {
|
|
domain, _ := strings.CutPrefix(r.Domain.String(), "ResourceDomain")
|
|
if r.Shared {
|
|
return fmt.Sprintf("%s:%s@R", domain, r.Key)
|
|
}
|
|
return fmt.Sprintf("%s:%s@X", domain, r.Key)
|
|
}
|
|
|
|
func (r ResourceKey) ShortString() string {
|
|
domain, _ := strings.CutPrefix(r.Domain.String(), "ResourceDomain")
|
|
if r.Shared {
|
|
return fmt.Sprintf("%s@R", domain)
|
|
}
|
|
return fmt.Sprintf("%s@X", domain)
|
|
}
|
|
|
|
// NewSharedClusterResourceKey creates a shared cluster resource key.
|
|
func NewSharedClusterResourceKey() ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainCluster,
|
|
Key: "",
|
|
Shared: true,
|
|
}
|
|
}
|
|
|
|
// NewExclusiveClusterResourceKey creates an exclusive cluster resource key.
|
|
func NewExclusiveClusterResourceKey() ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainCluster,
|
|
Key: "",
|
|
Shared: false,
|
|
}
|
|
}
|
|
|
|
// NewSharedCollectionNameResourceKey creates a shared collection name resource key.
|
|
func NewSharedCollectionNameResourceKey(dbName string, collectionName string) ResourceKey {
|
|
if dbName == "" {
|
|
dbName = util.DefaultDBName
|
|
}
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainCollectionName,
|
|
Key: fmt.Sprintf("%s:%s", dbName, collectionName),
|
|
Shared: true,
|
|
}
|
|
}
|
|
|
|
// NewExclusiveCollectionNameResourceKey creates an exclusive collection name resource key.
|
|
func NewExclusiveCollectionNameResourceKey(dbName string, collectionName string) ResourceKey {
|
|
if dbName == "" {
|
|
dbName = util.DefaultDBName
|
|
}
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainCollectionName,
|
|
Key: fmt.Sprintf("%s:%s", dbName, collectionName),
|
|
Shared: false,
|
|
}
|
|
}
|
|
|
|
// NewSharedDBNameResourceKey creates a shared db name resource key.
|
|
func NewSharedDBNameResourceKey(dbName string) ResourceKey {
|
|
if dbName == "" {
|
|
dbName = util.DefaultDBName
|
|
}
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainDBName,
|
|
Key: dbName,
|
|
Shared: true,
|
|
}
|
|
}
|
|
|
|
// NewExclusiveDBNameResourceKey creates an exclusive db name resource key.
|
|
func NewExclusiveDBNameResourceKey(dbName string) ResourceKey {
|
|
if dbName == "" {
|
|
dbName = util.DefaultDBName
|
|
}
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainDBName,
|
|
Key: dbName,
|
|
Shared: false,
|
|
}
|
|
}
|
|
|
|
// NewExclusivePrivilegeResourceKey creates an exclusive privilege resource key.
|
|
func NewExclusivePrivilegeResourceKey() ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainPrivilege,
|
|
Key: "",
|
|
Shared: false,
|
|
}
|
|
}
|
|
|
|
// NewSharedSnapshotNameResourceKey creates a shared snapshot name resource key.
|
|
func NewSharedSnapshotNameResourceKey(snapshotName string) ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainSnapshotName,
|
|
Key: snapshotName,
|
|
Shared: true,
|
|
}
|
|
}
|
|
|
|
// NewExclusiveSnapshotNameResourceKey creates an exclusive snapshot name resource key.
|
|
func NewExclusiveSnapshotNameResourceKey(snapshotName string) ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainSnapshotName,
|
|
Key: snapshotName,
|
|
Shared: false,
|
|
}
|
|
}
|
|
|
|
// Deprecated: NewImportJobIDResourceKey creates a key for import job resource.
|
|
func NewImportJobIDResourceKey(importJobID int64) ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainImportJobID,
|
|
Key: strconv.FormatInt(importJobID, 10),
|
|
}
|
|
}
|
|
|
|
// Deprecated: NewCollectionNameResourceKey creates a key for collection name resource.
|
|
func NewCollectionNameResourceKey(collectionName string) ResourceKey {
|
|
return ResourceKey{
|
|
Domain: messagespb.ResourceDomain_ResourceDomainCollectionName,
|
|
Key: collectionName,
|
|
}
|
|
}
|