admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 18:18:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix msgstream unsubscription (#16883)

Browse Source 
Signed-off-by: xiaofan-luan <xiaofan.luan@zilliz.com>
This commit is contained in:
Xiaofan 2022-05-10 19:43:52 +08:00 committed by GitHub
parent 34833680da
commit 000c5ff3de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/mq/msgstream/mq_msgstream.go
View File

@ -167,11 +167,13 @@ func (ms *mqMsgStream) AsConsumerWithPosition(channels []string, subName string,
ms.consumerChannels = append(ms.consumerChannels, channel) ms.consumerChannels = append(ms.consumerChannels, channel)
return nil return nil
} }
err := retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(5*time.Second)) // TODO if know the former subscribe is invalid, should we use pulsarctl to accelerate recovery speed
err := retry.Do(context.TODO(), fn, retry.Attempts(50), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(5*time.Second))
if err != nil { if err != nil {
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error() errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg) panic(errMsg)
} }
log.Info("Successfully create consumer", zap.String("channel", channel), zap.String("subname", subName))
} }
} }

6
internal/mq/msgstream/mqwrapper/pulsar/pulsar_consumer.go
View File

@ -115,6 +115,12 @@ func (pc *Consumer) Close() {
// Unsubscribe for the consumer // Unsubscribe for the consumer
fn := func() error { fn := func() error {
err := pc.c.Unsubscribe() err := pc.c.Unsubscribe()
if isPulsarError(err, pulsar.SubscriptionNotFound) || isPulsarError(err, pulsar.ConsumerNotFound) {
log.Warn("failed to find consumer, skip unsubscribe",
zap.String("subscription", pc.Subscription()),
zap.Error(err))
return nil
}
if err != nil { if err != nil {
return err return err
} }

9
internal/querycoord/meta.go
View File

@ -1121,6 +1121,7 @@ func (m *MetaReplica) getWatchedChannelsByNodeID(nodeID int64) *querypb.Unsubscr
// 1. find all the search/dmChannel/deltaChannel the node has watched // 1. find all the search/dmChannel/deltaChannel the node has watched
colID2DmChannels := make(map[UniqueID][]string) colID2DmChannels := make(map[UniqueID][]string)
colID2DeltaChannels := make(map[UniqueID][]string) colID2DeltaChannels := make(map[UniqueID][]string)
// TODO remove colID2QueryChannel since it's not used
colID2QueryChannel := make(map[UniqueID]string) colID2QueryChannel := make(map[UniqueID]string)
dmChannelInfos := m.getDmChannelInfosByNodeID(nodeID) dmChannelInfos := m.getDmChannelInfosByNodeID(nodeID)
// get dmChannel/search channel the node has watched // get dmChannel/search channel the node has watched
@ -1137,9 +1138,13 @@ func (m *MetaReplica) getWatchedChannelsByNodeID(nodeID int64) *querypb.Unsubscr
} }
} }
segmentInfos := m.getSegmentInfosByNode(nodeID) segmentInfos := m.getSegmentInfosByNode(nodeID)
// get delta/search channel the node has watched colIDs := make(map[UniqueID]bool)
// iterate through segments to find unique collection ids
for _, segmentInfo := range segmentInfos { for _, segmentInfo := range segmentInfos {
collectionID := segmentInfo.CollectionID colIDs[segmentInfo.CollectionID] = true
}
// get delta/search channel the node has watched
for collectionID := range colIDs {
if _, ok := colID2DeltaChannels[collectionID]; !ok { if _, ok := colID2DeltaChannels[collectionID]; !ok {
deltaChanelInfos, err := m.getDeltaChannelsByCollectionID(collectionID) deltaChanelInfos, err := m.getDeltaChannelsByCollectionID(collectionID)
if err != nil { if err != nil {