From fffbc1e3c1520fac30a30c45b9cb3b5189fbb700 Mon Sep 17 00:00:00 2001 From: wei liu Date: Thu, 25 Apr 2024 10:47:25 +0800 Subject: [PATCH] fix: make grow task stale when node become stopping (#32556) issue: #30816 #32526 pr: #31487 this PR fix that grow task stuck at stopping node Signed-off-by: Wei Liu --- internal/querycoordv2/task/scheduler.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/querycoordv2/task/scheduler.go b/internal/querycoordv2/task/scheduler.go index b395ed2ce2..fae48d5aa9 100644 --- a/internal/querycoordv2/task/scheduler.go +++ b/internal/querycoordv2/task/scheduler.go @@ -816,6 +816,10 @@ func (scheduler *taskScheduler) checkSegmentTaskStale(task *SegmentTask) error { for _, action := range task.Actions() { switch action.Type() { case ActionTypeGrow: + if ok, _ := scheduler.nodeMgr.IsStoppingNode(action.Node()); ok { + log.Warn("task stale due to node offline", zap.Int64("segment", task.segmentID)) + return merr.WrapErrNodeOffline(action.Node()) + } taskType := GetTaskType(task) var segment *datapb.SegmentInfo if taskType == TaskTypeMove || taskType == TaskTypeUpdate { @@ -860,6 +864,10 @@ func (scheduler *taskScheduler) checkChannelTaskStale(task *ChannelTask) error { for _, action := range task.Actions() { switch action.Type() { case ActionTypeGrow: + if ok, _ := scheduler.nodeMgr.IsStoppingNode(action.Node()); ok { + log.Warn("task stale due to node offline", zap.String("channel", task.Channel())) + return merr.WrapErrNodeOffline(action.Node()) + } if scheduler.targetMgr.GetDmChannel(task.collectionID, task.Channel(), meta.NextTargetFirst) == nil { log.Warn("the task is stale, the channel to subscribe not exists in targets", zap.String("channel", task.Channel()))