diff --git a/internal/querynodev2/delegator/delegator_data.go b/internal/querynodev2/delegator/delegator_data.go
index 09bd04b1bc..745db68618 100644
--- a/internal/querynodev2/delegator/delegator_data.go
+++ b/internal/querynodev2/delegator/delegator_data.go
@@ -435,6 +435,7 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
 			if err != nil {
 				return err
 			}
+			sd.collection.Ref(1)
 			sd.segmentManager.Put(ctx, segments.SegmentTypeSealed, l0Seg)
 			return nil
 		}