From 13beb5ccc0382356688780285e7c2e491a8a4f38 Mon Sep 17 00:00:00 2001 From: yah01 Date: Thu, 14 Dec 2023 18:28:38 +0800 Subject: [PATCH] fix: load gets stuck probably (#29191) we found the load got stuck probably, and reviewed the logs. the target observer seems not working, the reason is the taskDispatcher removes the task in a goroutine, and modifies the task status after committing the task into the goroutine pool, but this may happen after the task removed, which leads to the task will never be removed related #29086 Signed-off-by: yah01 --- internal/querycoordv2/observers/task_dispatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/querycoordv2/observers/task_dispatcher.go b/internal/querycoordv2/observers/task_dispatcher.go index 720d415422..29ede76b1b 100644 --- a/internal/querycoordv2/observers/task_dispatcher.go +++ b/internal/querycoordv2/observers/task_dispatcher.go @@ -96,12 +96,12 @@ func (d *taskDispatcher[K]) schedule(ctx context.Context) { case <-d.notifyCh: d.tasks.Range(func(k K, submitted bool) bool { if !submitted { + d.tasks.Insert(k, true) d.pool.Submit(func() (any, error) { d.taskRunner(ctx, k) d.tasks.Remove(k) return struct{}{}, nil }) - d.tasks.Insert(k, true) } return true })