diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go
index 3dd03bb99a..4b0dc67296 100644
--- a/internal/querynodev2/segments/segment.go
+++ b/internal/querynodev2/segments/segment.go
@@ -374,7 +374,7 @@ func (s *LocalSegment) LastDeltaTimestamp() uint64 {
 	return s.lastDeltaTimestamp.Load()
 }
 
-func (s *LocalSegment) AddIndex(fieldID int64, info *IndexedFieldInfo) {
+func (s *LocalSegment) addIndex(fieldID int64, info *IndexedFieldInfo) {
 	s.fieldIndexes.Insert(fieldID, info)
 }
 
@@ -1145,6 +1145,13 @@ func (s *LocalSegment) LoadIndexInfo(ctx context.Context, indexInfo *querypb.Fie
 		zap.Int64("fieldID", indexInfo.FieldID)); err != nil {
 		return err
 	}
+
+	s.addIndex(indexInfo.GetFieldID(), &IndexedFieldInfo{
+		FieldBinlog: &datapb.FieldBinlog{
+			FieldID: indexInfo.GetFieldID(),
+		},
+		IndexInfo: indexInfo,
+	})
 	log.Info("updateSegmentIndex done")
 
 	return nil
diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go
index 1268197dbf..60a043aee7 100644
--- a/internal/querynodev2/segments/segment_loader.go
+++ b/internal/querynodev2/segments/segment_loader.go
@@ -1093,8 +1093,6 @@ func (loader *segmentLoader) loadFieldsIndex(ctx context.Context,
 			zap.Int32("current_index_version", fieldInfo.IndexInfo.GetCurrentIndexVersion()),
 		)
 
-		segment.AddIndex(fieldID, fieldInfo)
-
 		// set average row data size of variable field
 		field, err := schemaHelper.GetFieldFromID(fieldID)
 		if err != nil {
@@ -1512,17 +1510,13 @@ func (loader *segmentLoader) LoadIndex(ctx context.Context, segment *LocalSegmen
 
 			fieldInfo, ok := fieldInfos[info.GetFieldID()]
 			if !ok {
-				return merr.WrapErrParameterInvalid("index info with corresponding  field info", "missing field info", strconv.FormatInt(fieldInfo.GetFieldID(), 10))
+				return merr.WrapErrParameterInvalid("index info with corresponding field info", "missing field info", strconv.FormatInt(fieldInfo.GetFieldID(), 10))
 			}
 			err := loader.loadFieldIndex(ctx, segment, info)
 			if err != nil {
 				log.Warn("failed to load index for segment", zap.Error(err))
 				return err
 			}
-			segment.AddIndex(info.FieldID, &IndexedFieldInfo{
-				IndexInfo:   info,
-				FieldBinlog: fieldInfo,
-			})
 		}
 		loader.notifyLoadFinish(loadInfo)
 	}