diff --git a/internal/querynodev2/delegator/delegator_data.go b/internal/querynodev2/delegator/delegator_data.go
index 5e36dc074d..caa197adba 100644
--- a/internal/querynodev2/delegator/delegator_data.go
+++ b/internal/querynodev2/delegator/delegator_data.go
@@ -954,12 +954,22 @@ func (sd *shardDelegator) ReleaseSegments(ctx context.Context, req *querypb.Rele
 			pkoracle.WithSegmentType(commonpb.SegmentState_Sealed),
 			pkoracle.WithWorkerID(targetNodeID),
 		)
+		if sd.idfOracle != nil {
+			for _, segment := range sealed {
+				sd.idfOracle.Remove(segment.SegmentID, commonpb.SegmentState_Sealed)
+			}
+		}
 	}
 	if len(growing) > 0 {
 		sd.pkOracle.Remove(
 			pkoracle.WithSegmentIDs(lo.Map(growing, func(entry SegmentEntry, _ int) int64 { return entry.SegmentID })...),
 			pkoracle.WithSegmentType(commonpb.SegmentState_Growing),
 		)
+		if sd.idfOracle != nil {
+			for _, segment := range growing {
+				sd.idfOracle.Remove(segment.SegmentID, commonpb.SegmentState_Growing)
+			}
+		}
 	}
 
 	var releaseErr error