diff --git a/internal/rootcoord/meta_table.go b/internal/rootcoord/meta_table.go
index f15f3f1457..5ef9c80d45 100644
--- a/internal/rootcoord/meta_table.go
+++ b/internal/rootcoord/meta_table.go
@@ -1009,10 +1009,10 @@ func (mt *MetaTable) AlignSegmentsMeta(collID, partID UniqueID, segIDs map[Uniqu
 		for segID := range segMap {
 			if _, ok := segIDs[segID]; !ok {
 				recycledSegIDs = append(recycledSegIDs, segID)
-			}
-			if idxID2segIndex, ok := mt.segID2IndexMeta[segID]; ok {
-				for _, segIndex := range idxID2segIndex {
-					recycledBuildIDs = append(recycledBuildIDs, segIndex.BuildID)
+				if idxID2segIndex, ok := mt.segID2IndexMeta[segID]; ok {
+					for _, segIndex := range idxID2segIndex {
+						recycledBuildIDs = append(recycledBuildIDs, segIndex.BuildID)
+					}
 				}
 			}
 		}
diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go
index dab6e7506f..69f5dd647a 100644
--- a/internal/rootcoord/root_coord.go
+++ b/internal/rootcoord/root_coord.go
@@ -502,11 +502,14 @@ func (c *Core) checkFlushedSegments(ctx context.Context) {
 			}
 			recycledSegIDs, recycledBuildIDs := c.MetaTable.AlignSegmentsMeta(collID, partID, segIDs)
 			log.Info("there buildIDs should be remove index", zap.Int64s("buildIDs", recycledBuildIDs))
-			if err := c.CallRemoveIndexService(ctx, recycledBuildIDs); err != nil {
-				log.Error("CallRemoveIndexService remove indexes on segments failed",
-					zap.Int64s("need dropped buildIDs", recycledBuildIDs), zap.Error(err))
-				continue
+			if len(recycledBuildIDs) > 0 {
+				if err := c.CallRemoveIndexService(ctx, recycledBuildIDs); err != nil {
+					log.Error("CallRemoveIndexService remove indexes on segments failed",
+						zap.Int64s("need dropped buildIDs", recycledBuildIDs), zap.Error(err))
+					continue
+				}
 			}
+
 			if err := c.MetaTable.RemoveSegments(collID, partID, recycledSegIDs); err != nil {
 				log.Warn("remove segments failed, wait to retry", zap.Int64("collID", collID), zap.Int64("partID", partID),
 					zap.Int64s("segIDs", recycledSegIDs), zap.Error(err))