From 695cc769df8bdd037ea53ea195bd0c4cbcaa51e7 Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:02:30 +0800 Subject: [PATCH] Fix watch event timeout when IO Pool busy (#23293) Signed-off-by: aoiasd --- configs/milvus.yaml | 2 +- internal/datanode/data_sync_service.go | 7 +++---- pkg/util/paramtable/component_param.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/configs/milvus.yaml b/configs/milvus.yaml index fd09b133da..8644794515 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -262,7 +262,7 @@ indexNode: dataCoord: channel: - watchTimeoutInterval: 30 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer. + watchTimeoutInterval: 120 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer. balanceSilentDuration: 300 # The duration before the channelBalancer on datacoord to run balanceInterval: 360 #The interval for the channelBalancer on datacoord to check balance status segment: diff --git a/internal/datanode/data_sync_service.go b/internal/datanode/data_sync_service.go index 190b342802..f8f3b9268e 100644 --- a/internal/datanode/data_sync_service.go +++ b/internal/datanode/data_sync_service.go @@ -199,6 +199,9 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo, tick return err } + //tickler will update addSegment progress to watchInfo + tickler.watch() + defer tickler.stop() futures := make([]*conc.Future[any], 0, len(unflushedSegmentInfos)+len(flushedSegmentInfos)) for _, us := range unflushedSegmentInfos { @@ -275,10 +278,6 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo, tick futures = append(futures, future) } - //tickler will update addSegment progress to watchInfo - tickler.watch() - defer tickler.stop() - err = conc.AwaitAll(futures...) if err != nil { return err diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 41e6755182..02359f615f 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -1797,7 +1797,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { p.WatchTimeoutInterval = ParamItem{ Key: "dataCoord.channel.watchTimeoutInterval", Version: "2.2.3", - DefaultValue: "30", + DefaultValue: "120", Doc: "Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer.", Export: true, }