admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 17:48:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix load index with empty file list (#26236)

Browse Source 
Signed-off-by: yah01 <yah2er0ne@outlook.com>
This commit is contained in:
yah01 2023-08-09 18:39:16 +08:00 committed by GitHub
parent a888606423
commit 889424b3f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/querycoordv2/checkers/index_checker.go
View File

@ -102,7 +102,9 @@ func (c *IndexChecker) checkReplica(ctx context.Context, collection *meta.Collec
continue continue
} }
for _, info := range infos { for _, info := range infos {
if missingFields.Contain(info.GetFieldID()) && info.GetEnableIndex() { if missingFields.Contain(info.GetFieldID()) &&
info.GetEnableIndex() &&
len(info.GetIndexFilePaths()) > 0 {
segmentsToUpdate.Insert(segment) segmentsToUpdate.Insert(segment)
} }
} }

1
internal/querycoordv2/checkers/index_checker_test.go
View File

@ -100,6 +100,7 @@ func (suite *IndexCheckerSuite) TestLoadIndex() {
FieldID: 101, FieldID: 101,
IndexID: 1000, IndexID: 1000,
EnableIndex: true, EnableIndex: true,
IndexFilePaths: []string{"index"},
}, },
}, nil) }, nil)

7
internal/querynodev2/segments/segment_loader.go
View File

@ -470,7 +470,7 @@ func (loader *segmentLoader) loadSegment(ctx context.Context,
if segment.Type() == SegmentTypeSealed { if segment.Type() == SegmentTypeSealed {
fieldID2IndexInfo := make(map[int64]*querypb.FieldIndexInfo) fieldID2IndexInfo := make(map[int64]*querypb.FieldIndexInfo)
for _, indexInfo := range loadInfo.IndexInfos { for _, indexInfo := range loadInfo.IndexInfos {
if len(indexInfo.IndexFilePaths) > 0 { if len(indexInfo.GetIndexFilePaths()) > 0 {
fieldID := indexInfo.FieldID fieldID := indexInfo.FieldID
fieldID2IndexInfo[fieldID] = indexInfo fieldID2IndexInfo[fieldID] = indexInfo
} }
@ -947,6 +947,11 @@ func (loader *segmentLoader) LoadIndex(ctx context.Context, segment *LocalSegmen
func(info *datapb.FieldBinlog) (int64, *datapb.FieldBinlog) { return info.GetFieldID(), info }) func(info *datapb.FieldBinlog) (int64, *datapb.FieldBinlog) { return info.GetFieldID(), info })
for _, info := range loadInfo.GetIndexInfos() { for _, info := range loadInfo.GetIndexInfos() {
if len(info.GetIndexFilePaths()) == 0 {
log.Warn("failed to add index for segment, index file list is empty, the segment may be too small")
return merr.WrapErrIndexNotFound("index file list empty")
}
fieldInfo, ok := fieldInfos[info.GetFieldID()] fieldInfo, ok := fieldInfos[info.GetFieldID()]
if !ok { if !ok {
return merr.WrapErrParameterInvalid("index info with corresponding field info", "missing field info", strconv.FormatInt(fieldInfo.GetFieldID(), 10)) return merr.WrapErrParameterInvalid("index info with corresponding field info", "missing field info", strconv.FormatInt(fieldInfo.GetFieldID(), 10))

20
internal/querynodev2/segments/segment_loader_test.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/initcore" "github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric" "github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/paramtable"
) )
@ -352,6 +353,25 @@ func (suite *SegmentLoaderSuite) TestLoadDeltaLogs() {
} }
} }
func (suite *SegmentLoaderSuite) TestLoadIndex() {
ctx := context.Background()
segment := &LocalSegment{}
loadInfo := &querypb.SegmentLoadInfo{
SegmentID: 1,
PartitionID: suite.partitionID,
CollectionID: suite.collectionID,
IndexInfos: []*querypb.FieldIndexInfo{
{
IndexFilePaths: []string{},
},
},
}
err := suite.loader.LoadIndex(ctx, segment, loadInfo)
suite.ErrorIs(err, merr.ErrIndexNotFound)
}
func (suite *SegmentLoaderSuite) TestLoadWithMmap() { func (suite *SegmentLoaderSuite) TestLoadWithMmap() {
key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key
paramtable.Get().Save(key, "/tmp/mmap-test") paramtable.Get().Save(key, "/tmp/mmap-test")