admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix index resource estimation (#32842)

Browse Source 
issue: #32820

---------

Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
Bingyi Sun 2024-05-10 11:53:30 +08:00 committed by GitHub
parent b7ef8da360
commit 5cbf081111
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 50 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/querynodev2/segments/segment_loader.go
View File

@ -1642,7 +1642,15 @@ func (loader *segmentLoader) LoadIndex(ctx context.Context, segment *LocalSegmen
indexInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info = typeutil.Clone(info)
info.BinlogPaths = nil
// remain binlog paths whose field id is in index infos to estimate resource usage correctly
indexFields := typeutil.NewSet(lo.Map(info.GetIndexInfos(), func(indexInfo *querypb.FieldIndexInfo, _ int) int64 { return indexInfo.GetFieldID() })...)
var binlogPaths []*datapb.FieldBinlog
for _, binlog := range info.GetBinlogPaths() {
if indexFields.Contain(binlog.GetFieldID()) {
binlogPaths = append(binlogPaths, binlog)
}
}
info.BinlogPaths = binlogPaths
info.Deltalogs = nil
info.Statslogs = nil
return info

41
internal/querynodev2/segments/segment_loader_test.go
View File

@ -42,6 +42,7 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
@ -491,6 +492,46 @@ func (suite *SegmentLoaderSuite) TestLoadIndex() {
suite.ErrorIs(err, merr.ErrIndexNotFound)
}
func (suite *SegmentLoaderSuite) TestLoadIndexWithLimitedResource() {
ctx := context.Background()
loadInfo := &querypb.SegmentLoadInfo{
SegmentID: 1,
PartitionID: suite.partitionID,
CollectionID: suite.collectionID,
IndexInfos: []*querypb.FieldIndexInfo{
{
FieldID: 1,
IndexFilePaths: []string{},
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.IndexINVERTED,
},
},
},
},
BinlogPaths: []*datapb.FieldBinlog{
{
FieldID: 1,
Binlogs: []*datapb.Binlog{
{
LogSize: 1000000000,
},
},
},
},
}
segment := &LocalSegment{
baseSegment: baseSegment{
loadInfo: atomic.NewPointer[querypb.SegmentLoadInfo](loadInfo),
},
}
paramtable.Get().QueryNodeCfg.DiskCapacityLimit.SwapTempValue("100000")
err := suite.loader.LoadIndex(ctx, segment, loadInfo, 0)
suite.Error(err)
}
func (suite *SegmentLoaderSuite) TestLoadWithMmap() {
key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key
paramtable.Get().Save(key, "/tmp/mmap-test")