enhance: [2.5] Accelerate dispatcher building (#42544)

Reduce check interval to accelerate dispatcher building.

issue: https://github.com/milvus-io/milvus/issues/42067

pr: https://github.com/milvus-io/milvus/pull/42500

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

configs/milvus.yaml

@@ -173,7 +173,7 @@ mq:
   pursuitBufferTime: 60 # pursuit mode buffer time in seconds
   mqBufSize: 16 # MQ client consumer buffer length
   dispatcher:
-    mergeCheckInterval: 1 # the interval time(in seconds) for dispatcher to check whether to merge
+    mergeCheckInterval: 0.1 # the interval time(in seconds) for dispatcher to check whether to merge
     targetBufSize: 16 # the lenth of channel buffer for targe
     maxTolerantLag: 3 # Default value: "3", the timeout(in seconds) that target sends msgPack
 

pkg/mq/msgdispatcher/dispatcher.go

@@ -240,7 +240,7 @@ func (d *Dispatcher) work() {
 			return
 		case pack := <-d.stream.Chan():
 			if pack == nil || len(pack.EndPositions) != 1 {
-				log.Error("consumed invalid msgPack")
+				log.Error("consumed invalid msgPack", zap.Any("pack", pack))
 				continue
 			}
 			d.curTs.Store(pack.EndPositions[0].GetTimestamp())

pkg/mq/msgdispatcher/manager.go

@@ -125,8 +125,8 @@ func (c *dispatcherManager) Close() {
 func (c *dispatcherManager) Run() {
 	log := log.With(zap.String("role", c.role), zap.Int64("nodeID", c.nodeID), zap.String("pchannel", c.pchannel))
 	log.Info("dispatcherManager is running...")
-	ticker1 := time.NewTicker(10 * time.Second)
-	ticker2 := time.NewTicker(paramtable.Get().MQCfg.MergeCheckInterval.GetAsDuration(time.Second))
+	ticker1 := time.NewTicker(30 * time.Second)
+	ticker2 := time.NewTicker(paramtable.Get().MQCfg.CheckInterval.GetAsDuration(time.Second))
 	defer ticker1.Stop()
 	defer ticker2.Stop()
 	for {

pkg/mq/msgdispatcher/manager_test.go

@@ -178,7 +178,7 @@ func TestManager(t *testing.T) {
 			d.curTs.Store(1000)
 		}
 
-		checkIntervalK := paramtable.Get().MQCfg.MergeCheckInterval.Key
+		checkIntervalK := paramtable.Get().MQCfg.CheckInterval.Key
 		paramtable.Get().Save(checkIntervalK, "0.01")
 		defer paramtable.Get().Reset(checkIntervalK)
 

pkg/util/paramtable/service_param.go

@@ -529,10 +529,10 @@ type MQConfig struct {
 	IgnoreBadPosition ParamItem `refreshable:"true"`
 
 	// msgdispatcher
-	MergeCheckInterval ParamItem `refreshable:"false"`
-	TargetBufSize      ParamItem `refreshable:"false"`
-	MaxTolerantLag     ParamItem `refreshable:"true"`
-	MaxPositionTsGap   ParamItem `refreshable:"true"`
+	CheckInterval    ParamItem `refreshable:"false"`
+	TargetBufSize    ParamItem `refreshable:"false"`
+	MaxTolerantLag   ParamItem `refreshable:"true"`
+	MaxPositionTsGap ParamItem `refreshable:"true"`
 }
 
 // Init initializes the MQConfig object with a BaseTable.
@@ -565,14 +565,14 @@ Valid values: [default, pulsar, kafka, rocksmq, natsmq]`,
 	}
 	p.TargetBufSize.Init(base.mgr)
 
-	p.MergeCheckInterval = ParamItem{
+	p.CheckInterval = ParamItem{
 		Key:          "mq.dispatcher.mergeCheckInterval",
 		Version:      "2.4.4",
-		DefaultValue: "1",
+		DefaultValue: "0.1",
 		Doc:          `the interval time(in seconds) for dispatcher to check whether to merge`,
 		Export:       true,
 	}
-	p.MergeCheckInterval.Init(base.mgr)
+	p.CheckInterval.Init(base.mgr)
 
 	p.MaxPositionTsGap = ParamItem{
 		Key:          "mq.dispatcher.maxPositionGapInMinutes",

pkg/util/paramtable/service_param_test.go

@@ -34,7 +34,7 @@ func TestServiceParam(t *testing.T) {
 
 	t.Run("test MQConfig", func(t *testing.T) {
 		Params := &SParams.MQCfg
-		assert.Equal(t, 1*time.Second, Params.MergeCheckInterval.GetAsDuration(time.Second))
+		assert.Equal(t, 100*time.Millisecond, Params.CheckInterval.GetAsDuration(time.Second))
 		assert.Equal(t, 16, Params.TargetBufSize.GetAsInt())
 		assert.Equal(t, 3*time.Second, Params.MaxTolerantLag.GetAsDuration(time.Second))
 		assert.Equal(t, 60*time.Minute, Params.MaxPositionTsGap.GetAsDuration(time.Minute))