From 8f4aaa2da8f6bfd759b1fb8c9bcbe3937222594b Mon Sep 17 00:00:00 2001 From: "yihao.dai" Date: Fri, 22 Sep 2023 18:31:25 +0800 Subject: [PATCH] Remove ValidateIndexedFieldsData (#27254) Signed-off-by: bigsheeper --- internal/querynodev2/segments/retrieve.go | 4 -- internal/querynodev2/segments/segment.go | 33 +----------- .../querynodev2/segments/segment_interface.go | 1 - internal/querynodev2/segments/segment_test.go | 54 ------------------- 4 files changed, 1 insertion(+), 91 deletions(-) diff --git a/internal/querynodev2/segments/retrieve.go b/internal/querynodev2/segments/retrieve.go index 7187a1e179..485c46f17b 100644 --- a/internal/querynodev2/segments/retrieve.go +++ b/internal/querynodev2/segments/retrieve.go @@ -101,10 +101,6 @@ func retrieveOnSegmentsWithStream(ctx context.Context, segments []Segment, segTy errs[i] = err return } - if err = seg.ValidateIndexedFieldsData(ctx, result); err != nil { - errs[i] = err - return - } if err = svr.Send(&internalpb.RetrieveResults{ Status: merr.Status(nil), diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go index f755182256..acb4a3fae9 100644 --- a/internal/querynodev2/segments/segment.go +++ b/internal/querynodev2/segments/segment.go @@ -43,12 +43,10 @@ import ( "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/segcorepb" - pkoracle "github.com/milvus-io/milvus/internal/querynodev2/pkoracle" + "github.com/milvus-io/milvus/internal/querynodev2/pkoracle" "github.com/milvus-io/milvus/internal/storage" - "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" - "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/timerecord" @@ -436,35 +434,6 @@ func (s *LocalSegment) GetFieldDataPath(index *IndexedFieldInfo, offset int64) ( return dataPath, offsetInBinlog } -func (s *LocalSegment) ValidateIndexedFieldsData(ctx context.Context, result *segcorepb.RetrieveResults) error { - log := log.Ctx(ctx).With( - zap.Int64("collectionID", s.Collection()), - zap.Int64("partitionID", s.Partition()), - zap.Int64("segmentID", s.ID()), - ) - - for _, fieldData := range result.FieldsData { - if !typeutil.IsVectorType(fieldData.GetType()) { - continue - } - if !s.ExistIndex(fieldData.FieldId) { - continue - } - if !s.HasRawData(fieldData.FieldId) { - index := s.GetIndex(fieldData.FieldId) - indexType, err := funcutil.GetAttrByKeyFromRepeatedKV(common.IndexTypeKey, index.IndexInfo.GetIndexParams()) - if err != nil { - return err - } - err = fmt.Errorf("vector output fields for %s index is not allowed", indexType) - log.Warn("validate fields failed", zap.Error(err)) - return err - } - } - - return nil -} - // -------------------------------------------------------------------------------------- interfaces for growing segment func (s *LocalSegment) preInsert(numOfRecords int) (int64, error) { /* diff --git a/internal/querynodev2/segments/segment_interface.go b/internal/querynodev2/segments/segment_interface.go index 601223d0a2..71f24cc90d 100644 --- a/internal/querynodev2/segments/segment_interface.go +++ b/internal/querynodev2/segments/segment_interface.go @@ -63,7 +63,6 @@ type Segment interface { // Read operations Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error) Retrieve(ctx context.Context, plan *RetrievePlan) (*segcorepb.RetrieveResults, error) - ValidateIndexedFieldsData(ctx context.Context, result *segcorepb.RetrieveResults) error Release() } diff --git a/internal/querynodev2/segments/segment_test.go b/internal/querynodev2/segments/segment_test.go index 46f4e3f34f..6c8b90e3e6 100644 --- a/internal/querynodev2/segments/segment_test.go +++ b/internal/querynodev2/segments/segment_test.go @@ -8,11 +8,8 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proto/querypb" - "github.com/milvus-io/milvus/internal/proto/segcorepb" storage "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/util/initcore" - "github.com/milvus-io/milvus/pkg/util/funcutil" - "github.com/milvus-io/milvus/pkg/util/metric" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -148,57 +145,6 @@ func (suite *SegmentSuite) TestHasRawData() { suite.True(has) } -func (suite *SegmentSuite) TestValidateIndexedFieldsData() { - result := &segcorepb.RetrieveResults{ - Ids: &schemapb.IDs{ - IdField: &schemapb.IDs_IntId{ - IntId: &schemapb.LongArray{ - Data: []int64{5, 4, 3, 2, 9, 8, 7, 6}, - }, - }, - }, - Offset: []int64{5, 4, 3, 2, 9, 8, 7, 6}, - FieldsData: []*schemapb.FieldData{ - genFieldData("int64 field", 100, schemapb.DataType_Int64, - []int64{5, 4, 3, 2, 9, 8, 7, 6}, 1), - genFieldData("float vector field", 101, schemapb.DataType_FloatVector, - []float32{5, 4, 3, 2, 9, 8, 7, 6}, 1), - }, - } - - // no index - err := suite.growing.ValidateIndexedFieldsData(context.Background(), result) - suite.NoError(err) - err = suite.sealed.ValidateIndexedFieldsData(context.Background(), result) - suite.NoError(err) - - // with index and has raw data - suite.sealed.AddIndex(101, &IndexedFieldInfo{ - IndexInfo: &querypb.FieldIndexInfo{ - FieldID: 101, - EnableIndex: true, - }, - }) - suite.True(suite.sealed.ExistIndex(101)) - err = suite.sealed.ValidateIndexedFieldsData(context.Background(), result) - suite.NoError(err) - - // index doesn't have index type - suite.sealed.Release() - suite.True(suite.sealed.ExistIndex(101)) - err = suite.sealed.ValidateIndexedFieldsData(context.Background(), result) - suite.Error(err) - - // with index but doesn't have raw data - index := suite.sealed.GetIndex(101) - _, indexParams := genIndexParams(IndexHNSW, metric.L2) - index.IndexInfo.IndexParams = funcutil.Map2KeyValuePair(indexParams) - suite.sealed.Release() - suite.True(suite.sealed.ExistIndex(101)) - err = suite.sealed.ValidateIndexedFieldsData(context.Background(), result) - suite.Error(err) -} - func (suite *SegmentSuite) TestCASVersion() { segment := suite.sealed