admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: stats log lost after disable stats log loading on flush (#36592)

Browse Source 
issue: #36555

Signed-off-by: chyezh <chyezh@outlook.com>
This commit is contained in:
Zhen Ye 2024-09-29 16:53:15 +08:00 committed by GitHub
parent a6545b2e29
commit a47abb2f2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 17 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
internal/flushcommon/pipeline/data_sync_service.go
View File

@ -132,6 +132,10 @@ func (dsService *DataSyncService) GetMetaCache() metacache.MetaCache {
return dsService.metacache return dsService.metacache
} }
func getMetaCacheForStreaming(initCtx context.Context, params *util.PipelineParams, info *datapb.ChannelWatchInfo, unflushed, flushed []*datapb.SegmentInfo) (metacache.MetaCache, error) {
return initMetaCache(initCtx, params.ChunkManager, info, nil, unflushed, flushed)
}
func getMetaCacheWithTickler(initCtx context.Context, params *util.PipelineParams, info *datapb.ChannelWatchInfo, tickler *util.Tickler, unflushed, flushed []*datapb.SegmentInfo) (metacache.MetaCache, error) { func getMetaCacheWithTickler(initCtx context.Context, params *util.PipelineParams, info *datapb.ChannelWatchInfo, tickler *util.Tickler, unflushed, flushed []*datapb.SegmentInfo) (metacache.MetaCache, error) {
tickler.SetTotal(int32(len(unflushed) + len(flushed))) tickler.SetTotal(int32(len(unflushed) + len(flushed)))
return initMetaCache(initCtx, params.ChunkManager, info, tickler, unflushed, flushed) return initMetaCache(initCtx, params.ChunkManager, info, tickler, unflushed, flushed)
@ -161,7 +165,7 @@ func initMetaCache(initCtx context.Context, chunkManager storage.ChunkManager, i
return nil, err return nil, err
} }
segmentPks.Insert(segment.GetID(), pkoracle.NewBloomFilterSet(stats...)) segmentPks.Insert(segment.GetID(), pkoracle.NewBloomFilterSet(stats...))
if !streamingutil.IsStreamingServiceEnabled() { if tickler != nil {
tickler.Inc() tickler.Inc()
} }
@ -180,8 +184,11 @@ func initMetaCache(initCtx context.Context, chunkManager storage.ChunkManager, i
} }
} }
// growing segments's stats should always be loaded, for generating merged pk bf.
loadSegmentStats("growing", unflushed) loadSegmentStats("growing", unflushed)
loadSegmentStats("sealed", flushed) if !(streamingutil.IsStreamingServiceEnabled() || paramtable.Get().DataNodeCfg.SkipBFStatsLoad.GetAsBool()) {
loadSegmentStats("sealed", flushed)
}
// use fetched segment info // use fetched segment info
info.Vchan.FlushedSegments = flushed info.Vchan.FlushedSegments = flushed
@ -344,22 +351,10 @@ func NewDataSyncService(initCtx context.Context, pipelineParams *util.PipelinePa
} }
} }
if paramtable.Get().DataNodeCfg.SkipBFStatsLoad.GetAsBool() { // init metaCache meta
// In SkipBFStatsLoad mode, flushed segments no longer maintain a bloom filter. if metaCache, err = getMetaCacheWithTickler(initCtx, pipelineParams, info, tickler, unflushedSegmentInfos, flushedSegmentInfos); err != nil {
// So, here we skip loading the bloom filter for flushed segments. return nil, err
info.Vchan.FlushedSegments = flushedSegmentInfos
info.Vchan.UnflushedSegments = unflushedSegmentInfos
metaCache = metacache.NewMetaCache(info, func(segment *datapb.SegmentInfo) pkoracle.PkStat {
return pkoracle.NewBloomFilterSet()
}, metacache.NoneBm25StatsFactory)
} else {
// init metaCache meta
metaCache, err = getMetaCacheWithTickler(initCtx, pipelineParams, info, tickler, unflushedSegmentInfos, flushedSegmentInfos)
if err != nil {
return nil, err
}
} }
return getServiceWithChannel(initCtx, pipelineParams, info, metaCache, unflushedSegmentInfos, flushedSegmentInfos, nil) return getServiceWithChannel(initCtx, pipelineParams, info, metaCache, unflushedSegmentInfos, flushedSegmentInfos, nil)
} }
@ -367,6 +362,7 @@ func NewStreamingNodeDataSyncService(initCtx context.Context, pipelineParams *ut
// recover segment checkpoints // recover segment checkpoints
var ( var (
err error err error
metaCache metacache.MetaCache
unflushedSegmentInfos []*datapb.SegmentInfo unflushedSegmentInfos []*datapb.SegmentInfo
flushedSegmentInfos []*datapb.SegmentInfo flushedSegmentInfos []*datapb.SegmentInfo
) )
@ -383,13 +379,10 @@ func NewStreamingNodeDataSyncService(initCtx context.Context, pipelineParams *ut
} }
} }
// In streaming service mode, flushed segments no longer maintain a bloom filter. // init metaCache meta
// So, here we skip loading the bloom filter for flushed segments. if metaCache, err = getMetaCacheForStreaming(initCtx, pipelineParams, info, unflushedSegmentInfos, flushedSegmentInfos); err != nil {
info.Vchan.UnflushedSegments = unflushedSegmentInfos return nil, err
metaCache := metacache.NewMetaCache(info, func(segment *datapb.SegmentInfo) pkoracle.PkStat { }
return pkoracle.NewBloomFilterSet()
}, metacache.NoneBm25StatsFactory)
return getServiceWithChannel(initCtx, pipelineParams, info, metaCache, unflushedSegmentInfos, flushedSegmentInfos, input) return getServiceWithChannel(initCtx, pipelineParams, info, metaCache, unflushedSegmentInfos, flushedSegmentInfos, input)
} }