fix: add lazy load field to mark segment load type (#31591)

issue: https://github.com/milvus-io/milvus/issues/31673

---------

Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
Bingyi Sun 2024-03-28 11:23:10 +08:00 committed by GitHub
parent c42492c0fd
commit 3d66670619
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 800 additions and 754 deletions

View File

@ -459,6 +459,47 @@ func (_c *MockSegment_InsertCount_Call) RunAndReturn(run func() int64) *MockSegm
return _c return _c
} }
// IsLazyLoad provides a mock function with given fields:
func (_m *MockSegment) IsLazyLoad() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// MockSegment_IsLazyLoad_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsLazyLoad'
type MockSegment_IsLazyLoad_Call struct {
*mock.Call
}
// IsLazyLoad is a helper method to define mock.On call
func (_e *MockSegment_Expecter) IsLazyLoad() *MockSegment_IsLazyLoad_Call {
return &MockSegment_IsLazyLoad_Call{Call: _e.mock.On("IsLazyLoad")}
}
func (_c *MockSegment_IsLazyLoad_Call) Run(run func()) *MockSegment_IsLazyLoad_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockSegment_IsLazyLoad_Call) Return(_a0 bool) *MockSegment_IsLazyLoad_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockSegment_IsLazyLoad_Call) RunAndReturn(run func() bool) *MockSegment_IsLazyLoad_Call {
_c.Call.Return(run)
return _c
}
// LastDeltaTimestamp provides a mock function with given fields: // LastDeltaTimestamp provides a mock function with given fields:
func (_m *MockSegment) LastDeltaTimestamp() uint64 { func (_m *MockSegment) LastDeltaTimestamp() uint64 {
ret := _m.Called() ret := _m.Called()

View File

@ -67,7 +67,7 @@ func retrieveOnSegments(ctx context.Context, mgr *Manager, segments []Segment, s
defer wg.Done() defer wg.Done()
var err error var err error
if seg.LoadStatus() == LoadStatusMeta { if seg.IsLazyLoad() {
err = mgr.DiskCache.Do(seg.ID(), retriever) err = mgr.DiskCache.Do(seg.ID(), retriever)
} else { } else {
err = retriever(seg) err = retriever(seg)

View File

@ -77,7 +77,7 @@ func searchSegments(ctx context.Context, mgr *Manager, segments []Segment, segTy
mu.Unlock() mu.Unlock()
} }
var err error var err error
if seg.LoadStatus() == LoadStatusMeta { if seg.IsLazyLoad() {
err = mgr.DiskCache.Do(seg.ID(), searcher) err = mgr.DiskCache.Do(seg.ID(), searcher)
} else { } else {
err = searcher(seg) err = searcher(seg)

View File

@ -89,6 +89,7 @@ type baseSegment struct {
segmentType SegmentType segmentType SegmentType
bloomFilterSet *pkoracle.BloomFilterSet bloomFilterSet *pkoracle.BloomFilterSet
loadInfo *querypb.SegmentLoadInfo loadInfo *querypb.SegmentLoadInfo
isLazyLoad bool
resourceUsageCache *atomic.Pointer[ResourceUsage] resourceUsageCache *atomic.Pointer[ResourceUsage]
} }
@ -192,6 +193,8 @@ func (s *baseSegment) ResourceUsageEstimate() ResourceUsage {
return *usage return *usage
} }
func (s *baseSegment) IsLazyLoad() bool { return s.isLazyLoad }
type FieldInfo struct { type FieldInfo struct {
datapb.FieldBinlog datapb.FieldBinlog
RowCount int64 RowCount int64

View File

@ -92,4 +92,5 @@ type Segment interface {
// Read operations // Read operations
Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error) Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error)
Retrieve(ctx context.Context, plan *RetrievePlan) (*segcorepb.RetrieveResults, error) Retrieve(ctx context.Context, plan *RetrievePlan) (*segcorepb.RetrieveResults, error)
IsLazyLoad() bool
} }

View File

@ -1033,6 +1033,7 @@ func (loader *segmentLoader) LoadSegment(ctx context.Context,
if segment.Type() == SegmentTypeSealed { if segment.Type() == SegmentTypeSealed {
if loadStatus == LoadStatusMeta { if loadStatus == LoadStatusMeta {
segment.baseSegment.isLazyLoad = true
segment.baseSegment.loadInfo = loadInfo segment.baseSegment.loadInfo = loadInfo
} }
if err := loader.loadSealedSegment(ctx, loadInfo, segment, collection, loadStatus); err != nil { if err := loader.loadSealedSegment(ctx, loadInfo, segment, collection, loadStatus); err != nil {