mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 17:18:35 +08:00
f94b04e642
Cherry-pick from master pr: #45061 #45488 #45803 #46017 #44991 #45132 #45723 #45726 #45798 #45897 #45918 #44998 This feature integrates the Storage V2 (Loon) FFI interface as a unified storage layer for segment loading and index building in Milvus. It enables manifest-based data access, replacing the traditional binlog-based approach with a more efficient columnar storage format. Key changes: ### Segment Self-Managed Loading Architecture - Move segment loading orchestration from Go layer to C++ segcore - Add NewSegmentWithLoadInfo() API for passing load info during segment creation - Implement SetLoadInfo() and Load() methods in SegmentInterface - Support parallel loading of indexed and non-indexed fields - Enable both sealed and growing segments to self-manage loading ### Storage V2 FFI Integration - Integrate milvus-storage library's FFI interface for packed columnar data - Add manifest path support throughout the data path (SegmentInfo, LoadInfo) - Implement ManifestReader for generating manifests from binlogs - Support zero-copy data exchange using Arrow C Data Interface - Add ToCStorageConfig() for Go-to-C storage config conversion ### Manifest-Based Index Building - Extend FileManagerContext to carry loon_ffi_properties - Implement GetFieldDatasFromManifest() using Arrow C Stream interface - Support manifest-based reading in DiskFileManagerImpl and MemFileManagerImpl - Add fallback to traditional segment insert files when manifest unavailable ### Compaction Pipeline Updates - Include manifest path in all compaction task builders (clustering, L0, mix) - Update BulkPackWriterV2 to return manifest path - Propagate manifest metadata through compaction pipeline ### Configuration & Protocol - Add common.storageV2.useLoonFFI config option (default: false) - Add manifest_path field to SegmentLoadInfo and related proto messages - Add manifest field to compaction segment messages ### Bug Fixes - Fix mmap settings not applied during segment load (key typo fix) - Populate index info after segment loading to prevent redundant load tasks - Fix memory corruption by removing premature transaction handle destruction Related issues: #44956, #45060, #39173 ## Individual Cherry-Picked Commits 1. **e1c923b5cc** - fix: apply mmap settings correctly during segment load (#46017) 2. **63b912370b** - enhance: use milvus-storage internal C++ Reader API for Loon FFI (#45897) 3. **bfc192faa5** - enhance: Resolve issues integrating loon FFI (#45918) 4. **fb18564631** - enhance: support manifest-based index building with Loon FFI reader (#45726) 5. **b9ec2392b9** - enhance: integrate StorageV2 FFI interface for manifest-based segment loading (#45798) 6. **66db3c32e6** - enhance: integrate Storage V2 FFI interface for unified storage access (#45723) 7. **ae789273ac** - fix: populate index info after segment loading to prevent redundant load tasks (#45803) 8. **49688b0be2** - enhance: Move segment loading logic from Go layer to segcore for self-managed loading (#45488) 9. **5b2df88bac** - enhance: [StorageV2] Integrate FFI interface for packed reader (#45132) 10. **91ff5706ac** - enhance: [StorageV2] add manifest path support for FFI integration (#44991) 11. **2192bb4a85** - enhance: add NewSegmentWithLoadInfo API to support segment self-managed loading (#45061) 12. **4296b01da0** - enhance: update delta log serialization APIs to integrate storage V2 (#44998) ## Technical Details ### Architecture Changes - **Before**: Go layer orchestrated segment loading, making multiple CGO calls - **After**: Segments autonomously manage loading in C++ layer with single entry point ### Storage Access Pattern - **Before**: Read individual binlog files through Go storage layer - **After**: Read manifest file that references packed columnar data via FFI ### Benefits - Reduced cross-language call overhead - Better resource management at C++ level - Improved I/O performance through batched streaming reads - Cleaner separation of concerns between Go and C++ layers - Foundation for proactive schema evolution handling --------- Signed-off-by: Ted Xu <ted.xu@zilliz.com> Signed-off-by: Congqi Xia <congqi.xia@zilliz.com> Co-authored-by: Ted Xu <ted.xu@zilliz.com>
123 lines
2.9 KiB
Protocol Buffer
123 lines
2.9 KiB
Protocol Buffer
syntax = "proto3";
|
|
package milvus.proto.segcore;
|
|
|
|
option go_package = "github.com/milvus-io/milvus/pkg/v2/proto/segcorepb";
|
|
import "schema.proto";
|
|
import "common.proto";
|
|
|
|
message Binlog {
|
|
int64 entries_num = 1;
|
|
uint64 timestamp_from = 2;
|
|
uint64 timestamp_to = 3;
|
|
string log_path = 4;
|
|
int64 log_size = 5;
|
|
int64 logID = 6;
|
|
int64 memory_size = 7;
|
|
}
|
|
|
|
message FieldBinlog {
|
|
int64 fieldID = 1;
|
|
repeated Binlog binlogs = 2;
|
|
repeated int64 child_fields = 3;
|
|
}
|
|
|
|
message TextIndexStats {
|
|
int64 fieldID = 1;
|
|
int64 version = 2;
|
|
repeated string files = 3;
|
|
int64 log_size = 4;
|
|
int64 memory_size = 5;
|
|
int64 buildID = 6;
|
|
}
|
|
|
|
message JsonKeyStats {
|
|
int64 fieldID = 1;
|
|
int64 version = 2;
|
|
repeated string files = 3;
|
|
int64 log_size = 4;
|
|
int64 memory_size = 5;
|
|
int64 buildID = 6;
|
|
int64 json_key_stats_data_format = 7;
|
|
}
|
|
|
|
message RetrieveResults {
|
|
schema.IDs ids = 1;
|
|
repeated int64 offset = 2;
|
|
repeated schema.FieldData fields_data = 3;
|
|
int64 all_retrieve_count = 4;
|
|
bool has_more_result = 5;
|
|
int64 scanned_remote_bytes = 6;
|
|
int64 scanned_total_bytes = 7;
|
|
}
|
|
|
|
message LoadFieldMeta {
|
|
int64 min_timestamp = 1;
|
|
int64 max_timestamp = 2;
|
|
int64 row_count = 3;
|
|
}
|
|
|
|
message LoadSegmentMeta {
|
|
// TODOs
|
|
repeated LoadFieldMeta metas = 1;
|
|
int64 total_size = 2;
|
|
}
|
|
|
|
message InsertRecord {
|
|
repeated schema.FieldData fields_data = 1;
|
|
int64 num_rows = 2;
|
|
}
|
|
|
|
message FieldIndexMeta {
|
|
int64 fieldID = 1;
|
|
int64 collectionID = 2;
|
|
string index_name = 3;
|
|
repeated common.KeyValuePair type_params = 4;
|
|
repeated common.KeyValuePair index_params = 5;
|
|
bool is_auto_index = 6;
|
|
repeated common.KeyValuePair user_index_params = 7;
|
|
}
|
|
|
|
message CollectionIndexMeta {
|
|
int64 maxIndexRowCount = 1;
|
|
repeated FieldIndexMeta index_metas = 2;
|
|
}
|
|
|
|
message FieldIndexInfo {
|
|
int64 fieldID = 1;
|
|
bool enable_index = 2 [deprecated = true];
|
|
string index_name = 3;
|
|
int64 indexID = 4;
|
|
int64 buildID = 5;
|
|
repeated common.KeyValuePair index_params = 6;
|
|
repeated string index_file_paths = 7;
|
|
int64 index_size = 8;
|
|
int64 index_version = 9;
|
|
int64 num_rows = 10;
|
|
int32 current_index_version = 11;
|
|
int64 index_store_version = 12;
|
|
}
|
|
|
|
message SegmentLoadInfo {
|
|
int64 segmentID = 1;
|
|
int64 partitionID = 2;
|
|
int64 collectionID = 3;
|
|
int64 dbID = 4;
|
|
int64 flush_time = 5;
|
|
repeated FieldBinlog binlog_paths = 6;
|
|
int64 num_of_rows = 7;
|
|
repeated FieldBinlog statslogs = 8;
|
|
repeated FieldBinlog deltalogs = 9;
|
|
repeated int64 compactionFrom = 10; // segmentIDs compacted from
|
|
repeated FieldIndexInfo index_infos = 11;
|
|
int64 segment_size = 12 [deprecated = true];
|
|
string insert_channel = 13;
|
|
int64 readableVersion = 14;
|
|
int64 storageVersion = 15;
|
|
bool is_sorted = 16;
|
|
map<int64, TextIndexStats> textStatsLogs = 17;
|
|
repeated FieldBinlog bm25logs = 18;
|
|
map<int64, JsonKeyStats> jsonKeyStatsLogs = 19;
|
|
common.LoadPriority priority = 20;
|
|
string manifest_path = 21;
|
|
}
|