From 5a8ddde92ae8a48ca4effa3f78f818ff46acf8f4 Mon Sep 17 00:00:00 2001 From: yah01 Date: Thu, 14 Dec 2023 16:56:38 +0800 Subject: [PATCH] fix: load gets stuck probably (#29191) (#29192) we found the load got stuck probably, and reviewed the logs. the target observer seems not working, the reason is the taskDispatcher removes the task in a goroutine, and modifies the task status after committing the task into the goroutine pool, but this may happen after the task removed, which leads to the task will never be removed related #29086 pr: #29191 Signed-off-by: yah01 --- internal/querycoordv2/observers/task_dispatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/querycoordv2/observers/task_dispatcher.go b/internal/querycoordv2/observers/task_dispatcher.go index 720d415422..29ede76b1b 100644 --- a/internal/querycoordv2/observers/task_dispatcher.go +++ b/internal/querycoordv2/observers/task_dispatcher.go @@ -96,12 +96,12 @@ func (d *taskDispatcher[K]) schedule(ctx context.Context) { case <-d.notifyCh: d.tasks.Range(func(k K, submitted bool) bool { if !submitted { + d.tasks.Insert(k, true) d.pool.Submit(func() (any, error) { d.taskRunner(ctx, k) d.tasks.Remove(k) return struct{}{}, nil }) - d.tasks.Insert(k, true) } return true })