From 0c868b654d33b34f754613eac320bab34b678e3d Mon Sep 17 00:00:00 2001 From: Bingyi Sun Date: Mon, 25 Mar 2024 14:29:07 +0800 Subject: [PATCH] fix: fix mmap enabled check in resource estimation (#31536) issue: https://github.com/milvus-io/milvus/issues/31569 Signed-off-by: sunby --- .../querynodev2/segments/load_index_info.go | 19 ++++++++++++------- .../querynodev2/segments/segment_loader.go | 5 ++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/querynodev2/segments/load_index_info.go b/internal/querynodev2/segments/load_index_info.go index 51721c8f9e..f35c63292e 100644 --- a/internal/querynodev2/segments/load_index_info.go +++ b/internal/querynodev2/segments/load_index_info.go @@ -70,19 +70,24 @@ func deleteLoadIndexInfo(info *LoadIndexInfo) { }).Await() } +func isIndexMmapEnable(indexInfo *querypb.FieldIndexInfo) bool { + enableMmap := common.IsMmapEnabled(indexInfo.IndexParams...) + if !enableMmap { + _, ok := funcutil.KeyValuePair2Map(indexInfo.IndexParams)[common.MmapEnabledKey] + indexType := datacoord.GetIndexType(indexInfo.IndexParams) + indexSupportMmap := indexparamcheck.IsMmapSupported(indexType) + enableMmap = !ok && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool() && indexSupportMmap + } + return enableMmap +} + func (li *LoadIndexInfo) appendLoadIndexInfo(ctx context.Context, indexInfo *querypb.FieldIndexInfo, collectionID int64, partitionID int64, segmentID int64, fieldType schemapb.DataType) error { fieldID := indexInfo.FieldID indexPaths := indexInfo.IndexFilePaths indexParams := funcutil.KeyValuePair2Map(indexInfo.IndexParams) - enableMmap := common.IsMmapEnabled(indexInfo.IndexParams...) - if !enableMmap { - _, ok := indexParams[common.MmapEnabledKey] - indexType := datacoord.GetIndexType(indexInfo.IndexParams) - indexSupportMmap := indexparamcheck.IsMmapSupported(indexType) - enableMmap = !ok && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool() && indexSupportMmap - } + enableMmap := isIndexMmapEnable(indexInfo) // as Knowhere reports error if encounter a unknown param, we need to delete it delete(indexParams, common.MmapEnabledKey) diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index 7a0b4f4ac1..388646396b 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -1457,8 +1457,9 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn for _, fieldBinlog := range loadInfo.BinlogPaths { fieldID := fieldBinlog.FieldID - mmapEnabled := common.IsFieldMmapEnabled(schema, fieldID) + var mmapEnabled bool if fieldIndexInfo, ok := vecFieldID2IndexInfo[fieldID]; ok { + mmapEnabled = isIndexMmapEnable(fieldIndexInfo) neededMemSize, neededDiskSize, err := getIndexAttrCache().GetIndexResourceUsage(fieldIndexInfo, multiplyFactor.memoryIndexUsageFactor) if err != nil { return nil, errors.Wrapf(err, "failed to get index size collection %d, segment %d, indexBuildID %d", @@ -1473,6 +1474,8 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn segmentDiskSize += neededDiskSize } } else { + mmapEnabled = common.IsFieldMmapEnabled(schema, fieldID) || + (!common.FieldHasMmapKey(schema, fieldID) && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool()) binlogSize := uint64(getBinlogDataSize(fieldBinlog)) if mmapEnabled { segmentDiskSize += binlogSize