fix: Reduce querycoord check node in replica interval for test (#45837)

issue: #45791

Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>

internal/querycoordv2/observers/replica_observer.go

@@ -177,6 +177,7 @@ func (ob *ReplicaObserver) checkNodesInReplica() {
 	// check all ro nodes, remove it from replica if all segment/channel has been moved
 	for _, collectionID := range collections {
 		replicas := ob.meta.ReplicaManager.GetByCollection(ctx, collectionID)
+		hasNodeRemoved := false
 		for _, replica := range replicas {
 			if enableChannelExclusiveMode && !replica.IsChannelExclusiveModeEnabled() {
 				// register channel for enable exclusive mode
@@ -217,9 +218,13 @@ func (ob *ReplicaObserver) checkNodesInReplica() {
 					zap.Error(err))
 				continue
 			}
+			hasNodeRemoved = true
 			logger.Info("all segment/channel has been removed from ro node, remove it from replica",
 				zap.Int64s("removedNodes", removeNodes),
 			)
 		}
+		if hasNodeRemoved {
+			utils.RecoverReplicaOfCollection(ctx, ob.meta, collectionID)
+		}
 	}
 }

tests/_helm/values/nightly/distributed-kafka

@@ -89,6 +89,8 @@ extraConfigFiles:
     queryNode:
       segcore:
         exprEvalBatchSize: 512
+    queryCoord:
+      checkNodeInReplicaInterval: 3
 metrics:
   serviceMonitor:
     enabled: true

tests/_helm/values/nightly/distributed-pulsar-mmap

@@ -98,6 +98,8 @@ extraConfigFiles:
         scalarField: true
         scalarIndex: true
         growingMmapEnabled: true
+    queryCoord:
+      checkNodeInReplicaInterval: 3
 metrics:
   serviceMonitor:
     enabled: true

tests/_helm/values/nightly/distributed-woodpecker

@@ -94,6 +94,8 @@ extraConfigFiles:
     queryNode:
       segcore:
         exprEvalBatchSize: 512
+    queryCoord:
+      checkNodeInReplicaInterval: 3
 metrics:
   serviceMonitor:
     enabled: true