mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 17:18:35 +08:00
f94b04e642
Cherry-pick from master pr: #45061 #45488 #45803 #46017 #44991 #45132 #45723 #45726 #45798 #45897 #45918 #44998 This feature integrates the Storage V2 (Loon) FFI interface as a unified storage layer for segment loading and index building in Milvus. It enables manifest-based data access, replacing the traditional binlog-based approach with a more efficient columnar storage format. Key changes: ### Segment Self-Managed Loading Architecture - Move segment loading orchestration from Go layer to C++ segcore - Add NewSegmentWithLoadInfo() API for passing load info during segment creation - Implement SetLoadInfo() and Load() methods in SegmentInterface - Support parallel loading of indexed and non-indexed fields - Enable both sealed and growing segments to self-manage loading ### Storage V2 FFI Integration - Integrate milvus-storage library's FFI interface for packed columnar data - Add manifest path support throughout the data path (SegmentInfo, LoadInfo) - Implement ManifestReader for generating manifests from binlogs - Support zero-copy data exchange using Arrow C Data Interface - Add ToCStorageConfig() for Go-to-C storage config conversion ### Manifest-Based Index Building - Extend FileManagerContext to carry loon_ffi_properties - Implement GetFieldDatasFromManifest() using Arrow C Stream interface - Support manifest-based reading in DiskFileManagerImpl and MemFileManagerImpl - Add fallback to traditional segment insert files when manifest unavailable ### Compaction Pipeline Updates - Include manifest path in all compaction task builders (clustering, L0, mix) - Update BulkPackWriterV2 to return manifest path - Propagate manifest metadata through compaction pipeline ### Configuration & Protocol - Add common.storageV2.useLoonFFI config option (default: false) - Add manifest_path field to SegmentLoadInfo and related proto messages - Add manifest field to compaction segment messages ### Bug Fixes - Fix mmap settings not applied during segment load (key typo fix) - Populate index info after segment loading to prevent redundant load tasks - Fix memory corruption by removing premature transaction handle destruction Related issues: #44956, #45060, #39173 ## Individual Cherry-Picked Commits 1. **e1c923b5cc** - fix: apply mmap settings correctly during segment load (#46017) 2. **63b912370b** - enhance: use milvus-storage internal C++ Reader API for Loon FFI (#45897) 3. **bfc192faa5** - enhance: Resolve issues integrating loon FFI (#45918) 4. **fb18564631** - enhance: support manifest-based index building with Loon FFI reader (#45726) 5. **b9ec2392b9** - enhance: integrate StorageV2 FFI interface for manifest-based segment loading (#45798) 6. **66db3c32e6** - enhance: integrate Storage V2 FFI interface for unified storage access (#45723) 7. **ae789273ac** - fix: populate index info after segment loading to prevent redundant load tasks (#45803) 8. **49688b0be2** - enhance: Move segment loading logic from Go layer to segcore for self-managed loading (#45488) 9. **5b2df88bac** - enhance: [StorageV2] Integrate FFI interface for packed reader (#45132) 10. **91ff5706ac** - enhance: [StorageV2] add manifest path support for FFI integration (#44991) 11. **2192bb4a85** - enhance: add NewSegmentWithLoadInfo API to support segment self-managed loading (#45061) 12. **4296b01da0** - enhance: update delta log serialization APIs to integrate storage V2 (#44998) ## Technical Details ### Architecture Changes - **Before**: Go layer orchestrated segment loading, making multiple CGO calls - **After**: Segments autonomously manage loading in C++ layer with single entry point ### Storage Access Pattern - **Before**: Read individual binlog files through Go storage layer - **After**: Read manifest file that references packed columnar data via FFI ### Benefits - Reduced cross-language call overhead - Better resource management at C++ level - Improved I/O performance through batched streaming reads - Cleaner separation of concerns between Go and C++ layers - Foundation for proactive schema evolution handling --------- Signed-off-by: Ted Xu <ted.xu@zilliz.com> Signed-off-by: Congqi Xia <congqi.xia@zilliz.com> Co-authored-by: Ted Xu <ted.xu@zilliz.com>
89 lines
2.7 KiB
Go
89 lines
2.7 KiB
Go
package segcore
|
|
|
|
/*
|
|
#cgo pkg-config: milvus_core
|
|
|
|
#include "common/type_c.h"
|
|
|
|
*/
|
|
import "C"
|
|
|
|
import "context"
|
|
|
|
// CSegment is the interface of a segcore segment.
|
|
// TODO: We should separate the interface of CGrowingSegment and CSealedSegment,
|
|
// Because they have different implementations, GrowingSegment will only be used at streamingnode, SealedSegment will only be used at querynode.
|
|
// But currently, we just use the same interface to represent them to keep compatible with querynode LocalSegment.
|
|
type CSegment interface {
|
|
GrowingSegment
|
|
|
|
SealedSegment
|
|
}
|
|
|
|
// GrowingSegment is the interface of a growing segment.
|
|
type GrowingSegment interface {
|
|
basicSegmentMethodSet
|
|
|
|
// Insert inserts data into the segment.
|
|
Insert(ctx context.Context, request *InsertRequest) (*InsertResult, error)
|
|
}
|
|
|
|
// SealedSegment is the interface of a sealed segment.
|
|
type SealedSegment interface {
|
|
basicSegmentMethodSet
|
|
|
|
// LoadFieldData loads field data into the segment.
|
|
LoadFieldData(ctx context.Context, request *LoadFieldDataRequest) (*LoadFieldDataResult, error)
|
|
|
|
// AddFieldDataInfo adds field data info into the segment.
|
|
AddFieldDataInfo(ctx context.Context, request *AddFieldDataInfoRequest) (*AddFieldDataInfoResult, error)
|
|
|
|
// DropIndex drops the index of the segment.
|
|
DropIndex(ctx context.Context, fieldID int64) error
|
|
|
|
DropJSONIndex(ctx context.Context, fieldID int64, nestedPath string) error
|
|
}
|
|
|
|
// basicSegmentMethodSet is the basic method set of a segment.
|
|
type basicSegmentMethodSet interface {
|
|
// ID returns the ID of the segment.
|
|
ID() int64
|
|
|
|
// RawPointer returns the raw pointer of the segment.
|
|
// TODO: should be removed in future.
|
|
RawPointer() CSegmentInterface
|
|
|
|
// RawPointer returns the raw pointer of the segment.
|
|
RowNum() int64
|
|
|
|
// MemSize returns the memory size of the segment.
|
|
MemSize() int64
|
|
|
|
// HasRawData checks if the segment has raw data.
|
|
HasRawData(fieldID int64) bool
|
|
|
|
// HasFieldData checks if the segment has field data.
|
|
HasFieldData(fieldID int64) bool
|
|
|
|
// Search requests a search on the segment.
|
|
Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error)
|
|
|
|
// Retrieve retrieves entities from the segment.
|
|
Retrieve(ctx context.Context, plan *RetrievePlan) (*RetrieveResult, error)
|
|
|
|
// RetrieveByOffsets retrieves entities from the segment by offsets.
|
|
RetrieveByOffsets(ctx context.Context, plan *RetrievePlanWithOffsets) (*RetrieveResult, error)
|
|
|
|
// Delete deletes data from the segment.
|
|
Delete(ctx context.Context, request *DeleteRequest) (*DeleteResult, error)
|
|
|
|
// FinishLoad wraps up the load process and let segcore do the leftover jobs.
|
|
FinishLoad() error
|
|
|
|
// Load invokes segment managed loading.
|
|
Load(ctx context.Context) error
|
|
|
|
// Release releases the segment.
|
|
Release()
|
|
}
|