admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix datanode graceful stop panic (#25932)

Browse Source 
See also: #25925

Signed-off-by: yangxuan <xuan.yang@zilliz.com>
This commit is contained in:
XuanYang-cn 2023-07-28 10:11:08 +08:00 committed by GitHub
parent 833674c1cb
commit 84253f255e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 28 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/datanode/data_node.go
View File

@ -113,6 +113,7 @@ type DataNode struct {
//call once
initOnce sync.Once
startOnce sync.Once
stopOnce sync.Once
wg sync.WaitGroup
sessionMu sync.Mutex // to fix data race
session *sessionutil.Session
@ -583,13 +584,12 @@ func (node *DataNode) ReadyToFlush() error {
// Stop will release DataNode resources and shutdown datanode
func (node *DataNode) Stop() error {
node.stopOnce.Do(func() {
node.cancel()
// https://github.com/milvus-io/milvus/issues/12282
node.UpdateStateCode(commonpb.StateCode_Abnormal)
node.flowgraphManager.dropAll()
node.flowgraphManager.stop()
node.flowgraphManager.close()
node.cancel()
node.wg.Wait()
node.eventManagerMap.Range(func(_ string, m *channelEventManager) bool {
m.Close()
return true
@ -601,16 +601,15 @@ func (node *DataNode) Stop() error {
}
if node.closer != nil {
err := node.closer.Close()
if err != nil {
return err
}
node.closer.Close()
}
if node.session != nil {
node.session.Stop()
}
node.wg.Wait()
})
return nil
}

3
internal/datanode/flow_graph_manager.go
View File

@ -61,7 +61,8 @@ func (fm *flowgraphManager) start() {
}
}
func (fm *flowgraphManager) stop() {
func (fm *flowgraphManager) close() {
fm.dropAll()
fm.closeOnce.Do(func() {
close(fm.closeCh)
})

2
internal/proxy/proxy.go
View File

@ -70,7 +70,7 @@ var rateCol *ratelimitutil.RateCollector
// Proxy of milvus
type Proxy struct {
ctx context.Context
cancel func()
cancel context.CancelFunc
wg sync.WaitGroup
initParams *internalpb.InitParams

4
internal/util/flowgraph/input_node.go
View File

@ -66,7 +66,9 @@ func (inNode *InputNode) Operate(in []Msg) []Msg {
if !ok {
log.Warn("input closed", zap.Any("input node", inNode.Name()))
if inNode.lastMsg != nil {
log.Info("trigger force sync", zap.Int64("collection", inNode.collectionID), zap.Any("position", inNode.lastMsg))
log.Info("trigger force sync",
zap.Int64("collection", inNode.collectionID),
zap.Any("position", inNode.lastMsg.EndPositions))
return []Msg{&MsgStreamMsg{
BaseMsg: NewBaseMsg(true),
tsMessages: []msgstream.TsMsg{},