diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go
index 68bb5608ec..ab3a800fbc 100644
--- a/internal/querynodev2/segments/segment_loader.go
+++ b/internal/querynodev2/segments/segment_loader.go
@@ -377,10 +377,6 @@ func (loader *segmentLoader) Load(ctx context.Context,
 			segment.SetBloomFilter(bfs)
 		}
 
-		if err = segment.FinishLoad(); err != nil {
-			return errors.Wrap(err, "At FinishLoad")
-		}
-
 		if segment.Level() != datapb.SegmentLevel_L0 {
 			loader.manager.Segment.Put(ctx, segmentType, segment)
 		}
@@ -963,6 +959,10 @@ func (loader *segmentLoader) loadSealedSegment(ctx context.Context, loadInfo *qu
 	}
 	loadRawDataSpan := tr.RecordSpan()
 
+	if err = segment.FinishLoad(); err != nil {
+		return errors.Wrap(err, "At FinishLoad")
+	}
+
 	// load text indexes.
 	for _, info := range textIndexes {
 		if err := segment.LoadTextIndex(ctx, info, schemaHelper); err != nil {
@@ -1043,6 +1043,9 @@ func (loader *segmentLoader) LoadSegment(ctx context.Context,
 		if err := segment.LoadMultiFieldData(ctx); err != nil {
 			return err
 		}
+		if err := segment.FinishLoad(); err != nil {
+			return errors.Wrap(err, "At FinishLoad")
+		}
 	}
 
 	// load statslog if it's growing segment