diff --git a/internal/querycoordv2/handlers.go b/internal/querycoordv2/handlers.go
index f5a74e1169..98c7b48769 100644
--- a/internal/querycoordv2/handlers.go
+++ b/internal/querycoordv2/handlers.go
@@ -85,7 +85,11 @@ func (s *Server) balanceSegments(ctx context.Context, req *querypb.LoadBalanceRe
 	dstNodeSet.Remove(srcNode)
 
 	toBalance := typeutil.NewSet[*meta.Segment]()
+	// Only balance segments in targets
 	segments := s.dist.SegmentDistManager.GetByNode(srcNode)
+	segments = lo.Filter(segments, func(segment *meta.Segment, _ int) bool {
+		return s.targetMgr.GetSegment(segment.GetID()) != nil
+	})
 	allSegments := make(map[int64]*meta.Segment)
 	for _, segment := range segments {
 		allSegments[segment.GetID()] = segment