mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
975c91df16
issue: #44358 Implement complete snapshot management system including creation, deletion, listing, description, and restoration capabilities across all system components. Key features: - Create snapshots for entire collections - Drop snapshots by name with proper cleanup - List snapshots with collection filtering - Describe snapshot details and metadata Components added/modified: - Client SDK with full snapshot API support and options - DataCoord snapshot service with metadata management - Proxy layer with task-based snapshot operations - Protocol buffer definitions for snapshot RPCs - Comprehensive unit tests with mockey framework - Integration tests for end-to-end validation Technical implementation: - Snapshot metadata storage in etcd with proper indexing - File-based snapshot data persistence in object storage - Garbage collection integration for snapshot cleanup - Error handling and validation across all operations - Thread-safe operations with proper locking mechanisms <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant/assumption: snapshots are immutable point‑in‑time captures identified by (collection, snapshot name/ID); etcd snapshot metadata is authoritative for lifecycle (PENDING → COMMITTED → DELETING) and per‑segment manifests live in object storage (Avro / StorageV2). GC and restore logic must see snapshotRefIndex loaded (snapshotMeta.IsRefIndexLoaded) before reclaiming or relying on segment/index files. - New capability added: full end‑to‑end snapshot subsystem — client SDK APIs (Create/Drop/List/Describe/Restore + restore job queries), DataCoord SnapshotWriter/Reader (Avro + StorageV2 manifests), snapshotMeta in meta, SnapshotManager orchestration (create/drop/describe/list/restore), copy‑segment restore tasks/inspector/checker, proxy & RPC surface, GC integration, and docs/tests — enabling point‑in‑time collection snapshots persisted to object storage and restorations orchestrated across components. - Logic removed/simplified and why: duplicated recursive compaction/delta‑log traversal and ad‑hoc lookup code were consolidated behind two focused APIs/owners (Handler.GetDeltaLogFromCompactTo for delta traversal and SnapshotManager/SnapshotReader for snapshot I/O). MixCoord/coordinator broker paths were converted to thin RPC proxies. This eliminates multiple implementations of the same traversal/lookup, reducing divergence and simplifying responsibility boundaries. - Why this does NOT introduce data loss or regressions: snapshot create/drop use explicit two‑phase semantics (PENDING → COMMIT/DELETING) with SnapshotWriter writing manifests and metadata before commit; GC uses snapshotRefIndex guards and IsRefIndexLoaded/GetSnapshotBySegment/GetSnapshotByIndex checks to avoid removing referenced files; restore flow pre‑allocates job IDs, validates resources (partitions/indexes), performs rollback on failure (rollbackRestoreSnapshot), and converts/updates segment/index metadata only after successful copy tasks. Extensive unit and integration tests exercise pending/deleting/GC/restore/error paths to ensure idempotence and protection against premature deletion. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Wei Liu <wei.liu@zilliz.com>
479 lines
16 KiB
JSON
479 lines
16 KiB
JSON
{
|
|
"packages": {
|
|
"msgpb": "github.com/milvus-io/milvus-proto/go-api/v2/msgpb",
|
|
"messagespb": "github.com/milvus-io/milvus/pkg/v2/proto/messagespb"
|
|
},
|
|
"consts": {
|
|
"FieldMaskDB": "db",
|
|
"FieldMaskCollectionName": "collection_name",
|
|
"FieldMaskCollectionDescription": "description",
|
|
"FieldMaskCollectionSchema": "schema",
|
|
"FieldMaskCollectionProperties": "properties",
|
|
"FieldMaskCollectionConsistencyLevel": "consistency_level"
|
|
},
|
|
"ExtraExportTypes": [
|
|
"messagespb.PartitionSegmentAssignment",
|
|
"messagespb.SegmentAssignment",
|
|
"messagespb.ManualFlushExtraResponse",
|
|
"messagespb.RoleBinding",
|
|
"messagespb.AlterLoadConfigOfAlterDatabase",
|
|
"messagespb.AlterLoadConfigOfAlterCollection",
|
|
"messagespb.AlterCollectionMessageUpdates",
|
|
"messagespb.CacheExpirations",
|
|
"messagespb.CacheExpiration",
|
|
"messagespb.LegacyProxyCollectionMetaCache"
|
|
],
|
|
"messageReflectInfoTable": [
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.TimeTickMessageHeader",
|
|
"BodyType": "msgpb.TimeTickMsg"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeTimeTick",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.InsertMessageHeader",
|
|
"BodyType": "msgpb.InsertRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeInsert",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DeleteMessageHeader",
|
|
"BodyType": "msgpb.DeleteRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDelete",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreateCollectionMessageHeader",
|
|
"BodyType": "msgpb.CreateCollectionRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreateCollection",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropCollectionMessageHeader",
|
|
"BodyType": "msgpb.DropCollectionRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropCollection",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.TruncateCollectionMessageHeader",
|
|
"BodyType": "messagespb.TruncateCollectionMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeTruncateCollection",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreatePartitionMessageHeader",
|
|
"BodyType": "msgpb.CreatePartitionRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreatePartition",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropPartitionMessageHeader",
|
|
"BodyType": "msgpb.DropPartitionRequest"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropPartition",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.ImportMessageHeader",
|
|
"BodyType": "msgpb.ImportMsg"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeImport",
|
|
"Version": 1
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreateSegmentMessageHeader",
|
|
"BodyType": "messagespb.CreateSegmentMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreateSegment",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.FlushMessageHeader",
|
|
"BodyType": "messagespb.FlushMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeFlush",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.ManualFlushMessageHeader",
|
|
"BodyType": "messagespb.ManualFlushMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeManualFlush",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterReplicateConfigMessageHeader",
|
|
"BodyType": "messagespb.AlterReplicateConfigMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterReplicateConfig",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.BeginTxnMessageHeader",
|
|
"BodyType": "messagespb.BeginTxnMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeBeginTxn",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CommitTxnMessageHeader",
|
|
"BodyType": "messagespb.CommitTxnMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCommitTxn",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.RollbackTxnMessageHeader",
|
|
"BodyType": "messagespb.RollbackTxnMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeRollbackTxn",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.TxnMessageHeader",
|
|
"BodyType": "messagespb.TxnMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeTxn",
|
|
"Version": 2
|
|
},
|
|
"NoUtilFunctions": true
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.SchemaChangeMessageHeader",
|
|
"BodyType": "messagespb.SchemaChangeMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeSchemaChange",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterCollectionMessageHeader",
|
|
"BodyType": "messagespb.AlterCollectionMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterCollection",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterLoadConfigMessageHeader",
|
|
"BodyType": "messagespb.AlterLoadConfigMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterLoadConfig",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropLoadConfigMessageHeader",
|
|
"BodyType": "messagespb.DropLoadConfigMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropLoadConfig",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreateDatabaseMessageHeader",
|
|
"BodyType": "messagespb.CreateDatabaseMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreateDatabase",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterDatabaseMessageHeader",
|
|
"BodyType": "messagespb.AlterDatabaseMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterDatabase",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropDatabaseMessageHeader",
|
|
"BodyType": "messagespb.DropDatabaseMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropDatabase",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterAliasMessageHeader",
|
|
"BodyType": "messagespb.AlterAliasMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterAlias",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropAliasMessageHeader",
|
|
"BodyType": "messagespb.DropAliasMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropAlias",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterUserMessageHeader",
|
|
"BodyType": "messagespb.AlterUserMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterUser",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropUserMessageHeader",
|
|
"BodyType": "messagespb.DropUserMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropUser",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterRoleMessageHeader",
|
|
"BodyType": "messagespb.AlterRoleMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterRole",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropRoleMessageHeader",
|
|
"BodyType": "messagespb.DropRoleMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropRole",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterUserRoleMessageHeader",
|
|
"BodyType": "messagespb.AlterUserRoleMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterUserRole",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropUserRoleMessageHeader",
|
|
"BodyType": "messagespb.DropUserRoleMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropUserRole",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterPrivilegeMessageHeader",
|
|
"BodyType": "messagespb.AlterPrivilegeMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterPrivilege",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropPrivilegeMessageHeader",
|
|
"BodyType": "messagespb.DropPrivilegeMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropPrivilege",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterPrivilegeGroupMessageHeader",
|
|
"BodyType": "messagespb.AlterPrivilegeGroupMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterPrivilegeGroup",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropPrivilegeGroupMessageHeader",
|
|
"BodyType": "messagespb.DropPrivilegeGroupMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropPrivilegeGroup",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.RestoreRBACMessageHeader",
|
|
"BodyType": "messagespb.RestoreRBACMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeRestoreRBAC",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterResourceGroupMessageHeader",
|
|
"BodyType": "messagespb.AlterResourceGroupMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterResourceGroup",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropResourceGroupMessageHeader",
|
|
"BodyType": "messagespb.DropResourceGroupMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropResourceGroup",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreateIndexMessageHeader",
|
|
"BodyType": "messagespb.CreateIndexMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreateIndex",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.AlterIndexMessageHeader",
|
|
"BodyType": "messagespb.AlterIndexMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeAlterIndex",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropIndexMessageHeader",
|
|
"BodyType": "messagespb.DropIndexMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropIndex",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.FlushAllMessageHeader",
|
|
"BodyType": "messagespb.FlushAllMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeFlushAll",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.CreateSnapshotMessageHeader",
|
|
"BodyType": "messagespb.CreateSnapshotMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeCreateSnapshot",
|
|
"Version": 2
|
|
}
|
|
},
|
|
{
|
|
"MessageSpecializedType": {
|
|
"HeaderType": "messagespb.DropSnapshotMessageHeader",
|
|
"BodyType": "messagespb.DropSnapshotMessageBody"
|
|
},
|
|
"MessageTypeWithVersion": {
|
|
"MessageType": "MessageTypeDropSnapshot",
|
|
"Version": 2
|
|
}
|
|
}
|
|
]
|
|
} |