diff --git a/internal/datacoord/garbage_collector_test.go b/internal/datacoord/garbage_collector_test.go index d6b995a6c4..2ff0cb0ce9 100644 --- a/internal/datacoord/garbage_collector_test.go +++ b/internal/datacoord/garbage_collector_test.go @@ -122,15 +122,18 @@ func Test_garbageCollector_scan(t *testing.T) { t.Run("key is reference", func(t *testing.T) { segReferManager := &SegmentReferenceManager{ etcdKV: etcdKV, - segmentsLock: map[UniqueID][]*SegmentLock{ - 2: { - { - segmentID: 2, - nodeID: 1, - locKey: "path", + segmentsLock: map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock{ + 1: { + 1: { + TaskID: 1, + NodeID: 1, + SegmentIDs: []UniqueID{2}, }, }, }, + segmentReferCnt: map[UniqueID]int{ + 2: 1, + }, } gc := newGarbageCollector(meta, segRefer, GcOption{ cli: cli, @@ -149,7 +152,7 @@ func Test_garbageCollector_scan(t *testing.T) { validateMinioPrefixElements(t, cli, bucketName, path.Join(rootPath, deltaLogPrefix), delta) validateMinioPrefixElements(t, cli, bucketName, path.Join(rootPath, `indexes`), others) - err = gc.segRefer.ReleaseSegmentsLock([]UniqueID{2}, 1) + err = gc.segRefer.ReleaseSegmentsLock(1, 1) assert.NoError(t, err) gc.close() }) diff --git a/internal/datacoord/segment_reference_manager.go b/internal/datacoord/segment_reference_manager.go index b124bf1c0c..94eca4a3a1 100644 --- a/internal/datacoord/segment_reference_manager.go +++ b/internal/datacoord/segment_reference_manager.go @@ -17,143 +17,135 @@ package datacoord import ( - "fmt" "path" "strconv" - "strings" "sync" - "go.uber.org/zap" - + "github.com/golang/protobuf/proto" "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/datapb" + "go.uber.org/zap" ) -type SegmentLock struct { - segmentID UniqueID - nodeID UniqueID - locKey string -} - type SegmentReferenceManager struct { etcdKV kv.BaseKV - segmentsLock map[UniqueID][]*SegmentLock - lock sync.RWMutex -} - -func parseLockKey(key string) (segID UniqueID, nodeID UniqueID, err error) { - ss := strings.Split(key, "/") - // segment lock key consists of at least "meta/segmentRefer/nodeID/segID" - if len(ss) < 4 { - return 0, 0, fmt.Errorf("segment lock key is invalid with %s", key) - } - segID, err = strconv.ParseInt(ss[len(ss)-1], 10, 64) - if err != nil { - return 0, 0, err - } - nodeID, err = strconv.ParseInt(ss[len(ss)-2], 10, 64) - return segID, nodeID, err + // taskID -> (nodeID -> segmentReferenceLock), taskID must be globally unique in a component + segmentsLock map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock + segmentReferCnt map[UniqueID]int + lock sync.RWMutex } func NewSegmentReferenceManager(etcdKV kv.BaseKV, onlineIDs []UniqueID) (*SegmentReferenceManager, error) { - log.Info("New segment reference manager") + log.Info("create a new segment reference manager") segReferManager := &SegmentReferenceManager{ - etcdKV: etcdKV, - segmentsLock: make(map[UniqueID][]*SegmentLock), + etcdKV: etcdKV, + segmentsLock: make(map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock), + segmentReferCnt: map[UniqueID]int{}, + lock: sync.RWMutex{}, } - keys, _, err := segReferManager.etcdKV.LoadWithPrefix(segmentReferPrefix) + _, values, err := segReferManager.etcdKV.LoadWithPrefix(segmentReferPrefix) if err != nil { log.Error("load segments lock from etcd failed", zap.Error(err)) return nil, err } - for _, key := range keys { - segID, nodeID, err := parseLockKey(key) - if err != nil { - log.Error("parse segment lock key failed", zap.String("lock key", key), zap.Error(err)) + for _, value := range values { + segReferLock := &datapb.SegmentReferenceLock{} + if err = proto.Unmarshal([]byte(value), segReferLock); err != nil { + log.Error("unmarshal segment reference lock failed", zap.Error(err)) return nil, err } - segLock := &SegmentLock{ - segmentID: segID, - nodeID: nodeID, - locKey: key, + if _, ok := segReferManager.segmentsLock[segReferLock.TaskID]; !ok { + segReferManager.segmentsLock[segReferLock.TaskID] = map[UniqueID]*datapb.SegmentReferenceLock{} + } + segReferManager.segmentsLock[segReferLock.TaskID][segReferLock.NodeID] = segReferLock + for _, segID := range segReferLock.SegmentIDs { + segReferManager.segmentReferCnt[segID]++ } - segReferManager.segmentsLock[segID] = append(segReferManager.segmentsLock[segID], segLock) } err = segReferManager.recoverySegReferManager(onlineIDs) if err != nil { - log.Error("Recovery segment reference manager failed", zap.Error(err)) + log.Error("recovery segment reference manager failed", zap.Error(err)) return nil, err } - log.Info("New segment reference manager successfully") + log.Info("create new segment reference manager successfully") return segReferManager, nil } -func (srm *SegmentReferenceManager) AddSegmentsLock(segIDs []UniqueID, nodeID UniqueID) error { +func generateLocKey(taskID, nodeID UniqueID) string { + return path.Join(segmentReferPrefix, strconv.FormatInt(taskID, 10), strconv.FormatInt(nodeID, 10)) +} + +// AddSegmentsLock adds a reference lock on segments to ensure the segments does not compaction during the reference period. +func (srm *SegmentReferenceManager) AddSegmentsLock(taskID int64, segIDs []UniqueID, nodeID UniqueID) error { srm.lock.Lock() defer srm.lock.Unlock() - log.Info("Add reference lock on segments", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) - locKVs := make(map[string]string) - segID2SegmentLock := make(map[UniqueID][]*SegmentLock) - for _, segID := range segIDs { - locKey := path.Join(segmentReferPrefix, strconv.FormatInt(nodeID, 10), strconv.FormatInt(segID, 10)) - locKVs[locKey] = strconv.FormatInt(nodeID, 10) - segLock := &SegmentLock{ - segmentID: segID, - nodeID: nodeID, - locKey: locKey, - } - segID2SegmentLock[segID] = append(segID2SegmentLock[segID], segLock) + log.Info("add reference lock on segments", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) + + segReferLock := &datapb.SegmentReferenceLock{ + TaskID: taskID, + NodeID: nodeID, + SegmentIDs: segIDs, } - if err := srm.etcdKV.MultiSave(locKVs); err != nil { - log.Error("AddSegmentsLock save segment lock to etcd failed", zap.Int64s("segIDs", segIDs), zap.Error(err)) + value, err := proto.Marshal(segReferLock) + if err != nil { + log.Error("AddSegmentsLock marshal failed", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID), + zap.Int64s("segIDs", segIDs), zap.Error(err)) return err } - for segID, segLocks := range segID2SegmentLock { - srm.segmentsLock[segID] = append(srm.segmentsLock[segID], segLocks...) + if err = srm.etcdKV.Save(generateLocKey(taskID, nodeID), string(value)); err != nil { + log.Error("AddSegmentsLock save segment lock to etcd failed", zap.Int64("taskID", taskID), + zap.Int64("nodeID", nodeID), zap.Int64s("segIDs", segIDs), zap.Error(err)) + return err } - log.Info("Add reference lock on segments successfully", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) + if _, ok := srm.segmentsLock[taskID]; !ok { + srm.segmentsLock[taskID] = map[UniqueID]*datapb.SegmentReferenceLock{} + } + srm.segmentsLock[taskID][nodeID] = segReferLock + for _, segID := range segIDs { + srm.segmentReferCnt[segID]++ + } + log.Info("add reference lock on segments successfully", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) return nil } -func (srm *SegmentReferenceManager) ReleaseSegmentsLock(segIDs []UniqueID, nodeID UniqueID) error { +func (srm *SegmentReferenceManager) ReleaseSegmentsLock(taskID int64, nodeID UniqueID) error { srm.lock.Lock() defer srm.lock.Unlock() - log.Info("Release reference lock on segments", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) - locKeys := make([]string, 0) - for _, segID := range segIDs { - for _, segLock := range srm.segmentsLock[segID] { - if segLock.nodeID == nodeID { - locKeys = append(locKeys, segLock.locKey) - } - } + log.Info("release reference lock by taskID", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID)) + if _, ok := srm.segmentsLock[taskID]; !ok { + log.Warn("taskID has no reference lock on segment", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID)) + return nil } - if err := srm.etcdKV.MultiRemove(locKeys); err != nil { - log.Error("Remove reference lock paths on segments failed", zap.Int64s("segIDs", segIDs), + + if _, ok := srm.segmentsLock[taskID][nodeID]; !ok { + log.Warn("taskID has no reference lock on segment with the nodeID", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID)) + return nil + } + + if err := srm.etcdKV.Remove(generateLocKey(taskID, nodeID)); err != nil { + log.Error("remove reference lock paths by taskID failed", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID), zap.Error(err)) return err } - for _, segID := range segIDs { - if _, ok := srm.segmentsLock[segID]; !ok { - continue - } - for i := 0; i < len(srm.segmentsLock[segID]); i++ { - segLock := srm.segmentsLock[segID][i] - if segLock.nodeID == nodeID { - srm.segmentsLock[segID] = append(srm.segmentsLock[segID][:i], srm.segmentsLock[segID][i+1:]...) - i-- - } - } - if len(srm.segmentsLock[segID]) == 0 { - delete(srm.segmentsLock, segID) + for _, segID := range srm.segmentsLock[taskID][nodeID].SegmentIDs { + srm.segmentReferCnt[segID]-- + if srm.segmentReferCnt[segID] <= 0 { + delete(srm.segmentReferCnt, segID) } } - log.Info("Release reference lock on segments successfully", zap.Int64s("segIDs", segIDs), zap.Int64("nodeID", nodeID)) + + delete(srm.segmentsLock[taskID], nodeID) + if len(srm.segmentsLock[taskID]) == 0 { + delete(srm.segmentsLock, taskID) + } + log.Info("release reference lock by taskID successfully", zap.Int64("taskID", taskID), zap.Int64("nodeID", nodeID)) return nil } @@ -161,62 +153,55 @@ func (srm *SegmentReferenceManager) ReleaseSegmentsLockByNodeID(nodeID UniqueID) srm.lock.Lock() defer srm.lock.Unlock() - log.Info("Release reference lock on segments by node", zap.Int64("nodeID", nodeID)) - locKeys := make([]string, 0) - for segID := range srm.segmentsLock { - for _, segLock := range srm.segmentsLock[segID] { - if segLock.nodeID == nodeID { - locKeys = append(locKeys, segLock.locKey) + log.Info("release reference lock on segments by node", zap.Int64("nodeID", nodeID)) + for taskID, segReferLock := range srm.segmentsLock { + if _, ok := segReferLock[nodeID]; !ok { + continue + } + // The reason for not using MultiRemove is to prevent too many keys. + if err := srm.etcdKV.Remove(generateLocKey(taskID, nodeID)); err != nil { + log.Warn("remove reference lock path by taskID failed, need to retry", zap.Int64("nodeID", nodeID), + zap.Int64("taskID", taskID), zap.Error(err)) + return err + } + for _, segID := range segReferLock[nodeID].SegmentIDs { + srm.segmentReferCnt[segID]-- + if srm.segmentReferCnt[segID] <= 0 { + delete(srm.segmentReferCnt, segID) } } - } - if err := srm.etcdKV.MultiRemove(locKeys); err != nil { - log.Error("Remove reference lock paths on segments by node failed", - zap.Int64("nodeID", nodeID), zap.Error(err)) - return err + delete(srm.segmentsLock[taskID], nodeID) + if len(srm.segmentsLock[taskID]) == 0 { + delete(srm.segmentsLock, taskID) + } } - for segID := range srm.segmentsLock { - for i := 0; i < len(srm.segmentsLock[segID]); i++ { - segLock := srm.segmentsLock[segID][i] - if segLock.nodeID == nodeID { - srm.segmentsLock[segID] = append(srm.segmentsLock[segID][:i], srm.segmentsLock[segID][i+1:]...) - i-- - } - } - if len(srm.segmentsLock[segID]) == 0 { - delete(srm.segmentsLock, segID) - } - } - log.Info("Release reference lock on segments by node successfully", zap.Int64("nodeID", nodeID)) + log.Info("release reference lock on segments by node successfully", zap.Int64("nodeID", nodeID)) return nil } func (srm *SegmentReferenceManager) recoverySegReferManager(nodeIDs []UniqueID) error { - log.Info("Recovery reference lock on segments by online nodes", zap.Int64s("online nodeIDs", nodeIDs)) + log.Info("recovery reference lock on segments by online nodes", zap.Int64s("online nodeIDs", nodeIDs)) + onlineIDs := make(map[UniqueID]struct{}) + for _, nodeID := range nodeIDs { + onlineIDs[nodeID] = struct{}{} + } offlineIDs := make(map[UniqueID]struct{}) - for segID := range srm.segmentsLock { - for _, segLock := range srm.segmentsLock[segID] { - alive := false - for _, nodeID := range nodeIDs { - if segLock.nodeID == nodeID { - alive = true - break - } - } - if !alive { - offlineIDs[segLock.nodeID] = struct{}{} + for _, segLock := range srm.segmentsLock { + for nodeID := range segLock { + if _, ok := onlineIDs[nodeID]; !ok { + offlineIDs[nodeID] = struct{}{} } } } for nodeID := range offlineIDs { if err := srm.ReleaseSegmentsLockByNodeID(nodeID); err != nil { - log.Error("Remove reference lock on segments by offline node failed", + log.Error("remove reference lock on segments by offline node failed", zap.Int64("offline nodeID", nodeID), zap.Error(err)) return err } } - log.Info("Recovery reference lock on segments by online nodes successfully", zap.Int64s("online nodeIDs", nodeIDs), + log.Info("recovery reference lock on segments by online nodes successfully", zap.Int64s("online nodeIDs", nodeIDs), zap.Any("offline nodeIDs", offlineIDs)) return nil } @@ -225,17 +210,8 @@ func (srm *SegmentReferenceManager) HasSegmentLock(segID UniqueID) bool { srm.lock.RLock() defer srm.lock.RUnlock() - _, ok := srm.segmentsLock[segID] - return ok -} - -func (srm *SegmentReferenceManager) GetHasReferLockSegmentIDs() []UniqueID { - srm.lock.RLock() - defer srm.lock.RUnlock() - - segIDs := make([]UniqueID, 0) - for segID := range srm.segmentsLock { - segIDs = append(segIDs, segID) + if _, ok := srm.segmentReferCnt[segID]; !ok { + return false } - return segIDs + return true } diff --git a/internal/datacoord/segment_reference_manager_test.go b/internal/datacoord/segment_reference_manager_test.go index ed8e9caec0..ee36b9ccaa 100644 --- a/internal/datacoord/segment_reference_manager_test.go +++ b/internal/datacoord/segment_reference_manager_test.go @@ -22,6 +22,9 @@ import ( "strconv" "testing" + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" @@ -42,62 +45,48 @@ func Test_SegmentReferenceManager(t *testing.T) { var err error var locKey string nodeID := int64(1) - locKey = path.Join(segmentReferPrefix, strconv.FormatInt(nodeID, 10), strconv.FormatInt(2, 10)) - err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) + taskID := int64(10) + locKey = path.Join(segmentReferPrefix, strconv.FormatInt(taskID, 10)) + segReferLock1 := &datapb.SegmentReferenceLock{ + TaskID: taskID, + NodeID: nodeID, + SegmentIDs: []UniqueID{1}, + } + value, err := proto.Marshal(segReferLock1) + assert.NoError(t, err) + err = etcdKV.Save(locKey, string(value)) assert.NoError(t, err) segRefer, err = NewSegmentReferenceManager(etcdKV, []UniqueID{nodeID}) assert.NoError(t, err) assert.NotNil(t, segRefer) - err = etcdKV.Remove(locKey) - assert.NoError(t, err) - locKey = path.Join(segmentReferPrefix, strconv.FormatInt(nodeID, 10), "segID") - err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) - assert.NoError(t, err) - segRefer, err = NewSegmentReferenceManager(etcdKV, []UniqueID{nodeID}) - assert.Error(t, err) - assert.Nil(t, segRefer) - err = etcdKV.Remove(locKey) - assert.NoError(t, err) - - locKey = path.Join(segmentReferPrefix, "nodeID", strconv.FormatInt(3, 10)) - err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) - assert.NoError(t, err) - segRefer, err = NewSegmentReferenceManager(etcdKV, []UniqueID{nodeID}) - assert.Error(t, err) - assert.Nil(t, segRefer) - err = etcdKV.Remove(locKey) - assert.NoError(t, err) - - locKey = path.Join(segmentReferPrefix, "nodeID") - err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) - assert.NoError(t, err) - segRefer, err = NewSegmentReferenceManager(etcdKV, nil) - assert.Error(t, err) - assert.Nil(t, segRefer) - err = etcdKV.Remove(locKey) - assert.NoError(t, err) - - locKey = path.Join(segmentReferPrefix, strconv.FormatInt(nodeID, 10), strconv.FormatInt(2, 10)) - err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) - assert.NoError(t, err) - segRefer, err = NewSegmentReferenceManager(etcdKV, nil) + segRefer, err = NewSegmentReferenceManager(etcdKV, []UniqueID{nodeID + 1}) assert.NoError(t, err) assert.NotNil(t, segRefer) - has := segRefer.HasSegmentLock(2) - assert.False(t, has) + err = etcdKV.Remove(locKey) + assert.NoError(t, err) + + locKey = path.Join(segmentReferPrefix, strconv.FormatInt(taskID, 10)) + err = etcdKV.Save(locKey, strconv.FormatInt(nodeID, 10)) + assert.NoError(t, err) + segRefer, err = NewSegmentReferenceManager(etcdKV, []UniqueID{nodeID}) + assert.Error(t, err) + assert.Nil(t, segRefer) + err = etcdKV.Remove(locKey) + assert.NoError(t, err) }) segIDs := []UniqueID{1, 2, 3, 4, 5} nodeID := UniqueID(1) + taskID := UniqueID(10) segRefer, err = NewSegmentReferenceManager(etcdKV, nil) assert.NoError(t, err) assert.NotNil(t, segRefer) var has bool t.Run("AddSegmentsLock", func(t *testing.T) { - err = segRefer.AddSegmentsLock(segIDs, nodeID) + err = segRefer.AddSegmentsLock(taskID, segIDs, nodeID) assert.NoError(t, err) for _, segID := range segIDs { @@ -107,7 +96,10 @@ func Test_SegmentReferenceManager(t *testing.T) { }) t.Run("ReleaseSegmentsLock", func(t *testing.T) { - err = segRefer.ReleaseSegmentsLock(segIDs, nodeID) + err = segRefer.ReleaseSegmentsLock(taskID, nodeID+1) + assert.NoError(t, err) + + err = segRefer.ReleaseSegmentsLock(taskID, nodeID) assert.NoError(t, err) for _, segID := range segIDs { @@ -115,7 +107,15 @@ func Test_SegmentReferenceManager(t *testing.T) { assert.False(t, has) } - err = segRefer.ReleaseSegmentsLock([]UniqueID{6}, nodeID) + taskID = UniqueID(11) + + err = segRefer.ReleaseSegmentsLock(taskID, nodeID) + assert.NoError(t, err) + + has = segRefer.HasSegmentLock(6) + assert.False(t, has) + + err = segRefer.ReleaseSegmentsLock(taskID, nodeID) assert.NoError(t, err) has = segRefer.HasSegmentLock(6) @@ -125,7 +125,8 @@ func Test_SegmentReferenceManager(t *testing.T) { t.Run("ReleaseSegmentsLockByNodeID", func(t *testing.T) { segIDs = []UniqueID{10, 11, 12, 13, 14, 15} nodeID = 2 - err = segRefer.AddSegmentsLock(segIDs, nodeID) + taskID = UniqueID(12) + err = segRefer.AddSegmentsLock(taskID, segIDs, nodeID) assert.NoError(t, err) for _, segID := range segIDs { @@ -133,7 +134,15 @@ func Test_SegmentReferenceManager(t *testing.T) { assert.True(t, has) } - err = segRefer.ReleaseSegmentsLockByNodeID(UniqueID(2)) + err = segRefer.ReleaseSegmentsLockByNodeID(nodeID) + assert.NoError(t, err) + + for _, segID := range segIDs { + has = segRefer.HasSegmentLock(segID) + assert.False(t, has) + } + + err = segRefer.ReleaseSegmentsLockByNodeID(nodeID) assert.NoError(t, err) for _, segID := range segIDs { @@ -147,7 +156,8 @@ func Test_SegmentReferenceManager(t *testing.T) { t.Run("RecoverySegReferManager", func(t *testing.T) { segIDs = []UniqueID{16, 17, 18, 19, 20} - err = segRefer.AddSegmentsLock(segIDs, UniqueID(3)) + taskID = UniqueID(13) + err = segRefer.AddSegmentsLock(taskID, segIDs, UniqueID(3)) assert.NoError(t, err) for _, segID := range segIDs { @@ -156,7 +166,7 @@ func Test_SegmentReferenceManager(t *testing.T) { } segIDs2 := []UniqueID{21, 22, 23, 24, 25} - err = segRefer.AddSegmentsLock(segIDs2, UniqueID(4)) + err = segRefer.AddSegmentsLock(taskID, segIDs2, UniqueID(4)) assert.NoError(t, err) for _, segID := range segIDs2 { @@ -188,7 +198,8 @@ func Test_SegmentReferenceManager(t *testing.T) { t.Run("GetHasReferLockSegmentIDs", func(t *testing.T) { segIDs = []UniqueID{26, 27, 28, 29, 30} - err = segRefer.AddSegmentsLock(segIDs, UniqueID(5)) + taskID = UniqueID(14) + err = segRefer.AddSegmentsLock(taskID, segIDs, UniqueID(5)) assert.NoError(t, err) for _, segID := range segIDs { @@ -196,10 +207,6 @@ func Test_SegmentReferenceManager(t *testing.T) { assert.True(t, has) } - segmentIDs := segRefer.GetHasReferLockSegmentIDs() - assert.Equal(t, 5, len(segmentIDs)) - assert.ElementsMatch(t, segIDs, segmentIDs) - err = segRefer.ReleaseSegmentsLockByNodeID(UniqueID(5)) assert.NoError(t, err) @@ -207,9 +214,6 @@ func Test_SegmentReferenceManager(t *testing.T) { has = segRefer.HasSegmentLock(segID) assert.False(t, has) } - - segIDs = segRefer.GetHasReferLockSegmentIDs() - assert.Equal(t, 0, len(segIDs)) }) } @@ -219,14 +223,14 @@ type etcdKVMock struct { Fail int } -func (em *etcdKVMock) MultiSave(data map[string]string) error { +func (em *etcdKVMock) Save(key, value string) error { if em.Fail > 0 { return errors.New("error occurred") } return nil } -func (em *etcdKVMock) MultiRemove(keys []string) error { +func (em *etcdKVMock) Remove(key string) error { if em.Fail > 0 { return errors.New("error occurred") } @@ -240,7 +244,13 @@ func (em *etcdKVMock) LoadWithPrefix(prefix string) ([]string, []string, error) if em.Fail > 1 { return []string{"key"}, []string{"value"}, nil } - return []string{"meta/segmentRefer/1/2"}, []string{"1"}, nil + referLock := &datapb.SegmentReferenceLock{ + TaskID: 1, + NodeID: 1, + SegmentIDs: []UniqueID{1, 2, 3}, + } + value, _ := proto.Marshal(referLock) + return []string{segmentReferPrefix + "/1/1"}, []string{string(value)}, nil } func TestSegmentReferenceManager_Error(t *testing.T) { @@ -268,28 +278,55 @@ func TestSegmentReferenceManager_Error(t *testing.T) { etcdKV: emKV, } + taskID := UniqueID(1) t.Run("AddSegmentsLock", func(t *testing.T) { - err := segRefer.AddSegmentsLock([]UniqueID{1}, 1) + err := segRefer.AddSegmentsLock(taskID, []UniqueID{1}, 1) assert.Error(t, err) }) t.Run("ReleaseSegmentsLock", func(t *testing.T) { - err := segRefer.ReleaseSegmentsLock([]UniqueID{1}, 1) + nodeID := UniqueID(1) + segRefer = &SegmentReferenceManager{ + etcdKV: emKV, + segmentsLock: map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock{ + taskID: { + nodeID: { + TaskID: taskID, + NodeID: nodeID, + SegmentIDs: []UniqueID{1, 2, 3}, + }, + }, + }, + } + err := segRefer.ReleaseSegmentsLock(taskID, 1) assert.Error(t, err) }) t.Run("ReleaseSegmentsLockByNodeID", func(t *testing.T) { - err := segRefer.ReleaseSegmentsLockByNodeID(1) + nodeID := UniqueID(1) + segRefer = &SegmentReferenceManager{ + etcdKV: emKV, + segmentsLock: map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock{ + taskID: { + nodeID: { + TaskID: taskID, + NodeID: nodeID, + SegmentIDs: []UniqueID{1, 2, 3}, + }, + }, + }, + } + err := segRefer.ReleaseSegmentsLockByNodeID(nodeID) assert.Error(t, err) }) t.Run("recoverySegReferManager", func(t *testing.T) { - segRefer.segmentsLock = map[UniqueID][]*SegmentLock{ + segRefer.segmentsLock = map[UniqueID]map[UniqueID]*datapb.SegmentReferenceLock{ 2: { - { - segmentID: 2, - nodeID: 2, - locKey: "1/2/3", + 3: { + TaskID: 2, + NodeID: 3, + SegmentIDs: []UniqueID{1, 2, 3}, }, }, } diff --git a/internal/datacoord/server_test.go b/internal/datacoord/server_test.go index 8df1926f87..c024d4dfcd 100644 --- a/internal/datacoord/server_test.go +++ b/internal/datacoord/server_test.go @@ -2741,8 +2741,8 @@ func TestDataCoord_Import(t *testing.T) { closeTestServer(t, svr) status, err := svr.ReleaseSegmentLock(context.TODO(), &datapb.ReleaseSegmentLockRequest{ - SegmentIDs: []UniqueID{1, 2}, - NodeID: UniqueID(1), + TaskID: UniqueID(1), + NodeID: UniqueID(1), }) assert.NoError(t, err) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode()) diff --git a/internal/datacoord/services.go b/internal/datacoord/services.go index 58f1669298..664a1ca927 100644 --- a/internal/datacoord/services.go +++ b/internal/datacoord/services.go @@ -1112,7 +1112,7 @@ func (s *Server) AcquireSegmentLock(ctx context.Context, req *datapb.AcquireSegm return resp, nil } - err = s.segReferManager.AddSegmentsLock(req.SegmentIDs, req.NodeID) + err = s.segReferManager.AddSegmentsLock(req.TaskID, req.SegmentIDs, req.NodeID) if err != nil { log.Warn("Add reference lock on segments failed", zap.Int64s("segIDs", req.SegmentIDs), zap.Error(err)) resp.Reason = err.Error() @@ -1122,7 +1122,7 @@ func (s *Server) AcquireSegmentLock(ctx context.Context, req *datapb.AcquireSegm if !hasSegments || err != nil { log.Error("AcquireSegmentLock failed, try to release reference lock", zap.Error(err)) if err2 := retry.Do(ctx, func() error { - return s.segReferManager.ReleaseSegmentsLock(req.SegmentIDs, req.NodeID) + return s.segReferManager.ReleaseSegmentsLock(req.TaskID, req.NodeID) }, retry.Attempts(100)); err2 != nil { panic(err) } @@ -1145,9 +1145,9 @@ func (s *Server) ReleaseSegmentLock(ctx context.Context, req *datapb.ReleaseSegm return resp, nil } - err := s.segReferManager.ReleaseSegmentsLock(req.SegmentIDs, req.NodeID) + err := s.segReferManager.ReleaseSegmentsLock(req.TaskID, req.NodeID) if err != nil { - log.Error("DataCoord ReleaseSegmentLock failed", zap.Int64s("segmentIDs", req.SegmentIDs), zap.Int64("nodeID", req.NodeID), + log.Error("DataCoord ReleaseSegmentLock failed", zap.Int64("taskID", req.TaskID), zap.Int64("nodeID", req.NodeID), zap.Error(err)) resp.Reason = err.Error() return resp, nil diff --git a/internal/distributed/rootcoord/service_test.go b/internal/distributed/rootcoord/service_test.go index 66f5a2da58..7c73d4c802 100644 --- a/internal/distributed/rootcoord/service_test.go +++ b/internal/distributed/rootcoord/service_test.go @@ -229,10 +229,10 @@ func TestGrpcService(t *testing.T) { core.CallImportService = func(ctx context.Context, req *datapb.ImportTaskRequest) *datapb.ImportTaskResponse { return nil } - core.CallAddSegRefLock = func(context.Context, []int64) error { + core.CallAddSegRefLock = func(context.Context, int64, []int64) error { return nil } - core.CallReleaseSegRefLock = func(context.Context, []int64) error { + core.CallReleaseSegRefLock = func(context.Context, int64, []int64) error { return nil } diff --git a/internal/indexcoord/index_coord.go b/internal/indexcoord/index_coord.go index bbe5cef1c8..ab282e7dc0 100644 --- a/internal/indexcoord/index_coord.go +++ b/internal/indexcoord/index_coord.go @@ -853,7 +853,7 @@ func (i *IndexCoord) watchMetaLoop() { log.Debug("This task has finished", zap.Int64("indexBuildID", indexBuildID), zap.Int64("Finish by IndexNode", indexMeta.NodeID), zap.Int64("The version of the task", indexMeta.Version)) - if err = i.tryReleaseSegmentReferLock(ctx, []UniqueID{indexMeta.Req.SegmentID}); err != nil { + if err = i.tryReleaseSegmentReferLock(ctx, indexBuildID, []UniqueID{indexMeta.Req.SegmentID}); err != nil { panic(err) } i.nodeManager.pq.IncPriority(indexMeta.NodeID, -1) @@ -873,8 +873,10 @@ func (i *IndexCoord) watchMetaLoop() { } } -func (i *IndexCoord) tryAcquireSegmentReferLock(ctx context.Context, segIDs []UniqueID) error { +func (i *IndexCoord) tryAcquireSegmentReferLock(ctx context.Context, buildID UniqueID, segIDs []UniqueID) error { + // IndexCoord use buildID instead of taskID. status, err := i.dataCoordClient.AcquireSegmentLock(ctx, &datapb.AcquireSegmentLockRequest{ + TaskID: buildID, NodeID: i.session.ServerID, SegmentIDs: segIDs, }) @@ -891,11 +893,11 @@ func (i *IndexCoord) tryAcquireSegmentReferLock(ctx context.Context, segIDs []Un return nil } -func (i *IndexCoord) tryReleaseSegmentReferLock(ctx context.Context, segIDs []UniqueID) error { +func (i *IndexCoord) tryReleaseSegmentReferLock(ctx context.Context, buildID UniqueID, segIDs []UniqueID) error { releaseLock := func() error { status, err := i.dataCoordClient.ReleaseSegmentLock(ctx, &datapb.ReleaseSegmentLockRequest{ - NodeID: i.session.ServerID, - SegmentIDs: segIDs, + TaskID: buildID, + NodeID: i.session.ServerID, }) if err != nil { return err @@ -964,7 +966,7 @@ func (i *IndexCoord) assignTaskLoop() { for index, meta := range metas { indexBuildID := meta.indexMeta.IndexBuildID segID := meta.indexMeta.Req.SegmentID - if err := i.tryAcquireSegmentReferLock(ctx, []UniqueID{segID}); err != nil { + if err := i.tryAcquireSegmentReferLock(ctx, indexBuildID, []UniqueID{segID}); err != nil { log.Warn("IndexCoord try to acquire segment reference lock failed, maybe this segment has been compacted", zap.Int64("segID", segID), zap.Int64("buildID", indexBuildID), zap.Error(err)) continue diff --git a/internal/indexcoord/index_coord_test.go b/internal/indexcoord/index_coord_test.go index ab4dfdd0da..0dfd5ddb41 100644 --- a/internal/indexcoord/index_coord_test.go +++ b/internal/indexcoord/index_coord_test.go @@ -427,7 +427,7 @@ func Test_tryAcquireSegmentReferLock(t *testing.T) { ic.chunkManager = cmm t.Run("success", func(t *testing.T) { - err := ic.tryAcquireSegmentReferLock(context.Background(), []UniqueID{1}) + err := ic.tryAcquireSegmentReferLock(context.Background(), 1, []UniqueID{1}) assert.Nil(t, err) }) @@ -437,7 +437,7 @@ func Test_tryAcquireSegmentReferLock(t *testing.T) { Fail: false, } ic.dataCoordClient = dcmE - err := ic.tryAcquireSegmentReferLock(context.Background(), []UniqueID{1}) + err := ic.tryAcquireSegmentReferLock(context.Background(), 1, []UniqueID{1}) assert.Error(t, err) }) @@ -447,7 +447,7 @@ func Test_tryAcquireSegmentReferLock(t *testing.T) { Fail: true, } ic.dataCoordClient = dcmF - err := ic.tryAcquireSegmentReferLock(context.Background(), []UniqueID{1}) + err := ic.tryAcquireSegmentReferLock(context.Background(), 1, []UniqueID{1}) assert.Error(t, err) }) } @@ -466,7 +466,7 @@ func Test_tryReleaseSegmentReferLock(t *testing.T) { ic.dataCoordClient = dcm t.Run("success", func(t *testing.T) { - err := ic.tryReleaseSegmentReferLock(context.Background(), []UniqueID{1}) + err := ic.tryReleaseSegmentReferLock(context.Background(), 1, []UniqueID{1}) assert.NoError(t, err) }) } diff --git a/internal/proto/data_coord.proto b/internal/proto/data_coord.proto index 0a0513a3af..1767ac7c7d 100644 --- a/internal/proto/data_coord.proto +++ b/internal/proto/data_coord.proto @@ -187,12 +187,14 @@ message AcquireSegmentLockRequest { common.MsgBase base = 1; int64 nodeID = 2; repeated int64 segmentIDs = 3; + int64 taskID = 4; } message ReleaseSegmentLockRequest { common.MsgBase base = 1; int64 nodeID = 2; repeated int64 segmentIDs = 3; + int64 taskID = 4; } message VchannelInfo { @@ -526,3 +528,9 @@ message AddSegmentRequest { int64 partition_id = 5; int64 row_num = 6; } + +message SegmentReferenceLock { + int64 taskID = 1; + int64 nodeID = 2; + repeated int64 segmentIDs = 3; +} diff --git a/internal/proto/datapb/data_coord.pb.go b/internal/proto/datapb/data_coord.pb.go index 9a583913c5..38bb707f71 100644 --- a/internal/proto/datapb/data_coord.pb.go +++ b/internal/proto/datapb/data_coord.pb.go @@ -1128,6 +1128,7 @@ type AcquireSegmentLockRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` SegmentIDs []int64 `protobuf:"varint,3,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"` + TaskID int64 `protobuf:"varint,4,opt,name=taskID,proto3" json:"taskID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1179,10 +1180,18 @@ func (m *AcquireSegmentLockRequest) GetSegmentIDs() []int64 { return nil } +func (m *AcquireSegmentLockRequest) GetTaskID() int64 { + if m != nil { + return m.TaskID + } + return 0 +} + type ReleaseSegmentLockRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` SegmentIDs []int64 `protobuf:"varint,3,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"` + TaskID int64 `protobuf:"varint,4,opt,name=taskID,proto3" json:"taskID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1234,6 +1243,13 @@ func (m *ReleaseSegmentLockRequest) GetSegmentIDs() []int64 { return nil } +func (m *ReleaseSegmentLockRequest) GetTaskID() int64 { + if m != nil { + return m.TaskID + } + return 0 +} + type VchannelInfo struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` ChannelName string `protobuf:"bytes,2,opt,name=channelName,proto3" json:"channelName,omitempty"` @@ -4004,6 +4020,61 @@ func (m *AddSegmentRequest) GetRowNum() int64 { return 0 } +type SegmentReferenceLock struct { + TaskID int64 `protobuf:"varint,1,opt,name=taskID,proto3" json:"taskID,omitempty"` + NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` + SegmentIDs []int64 `protobuf:"varint,3,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SegmentReferenceLock) Reset() { *m = SegmentReferenceLock{} } +func (m *SegmentReferenceLock) String() string { return proto.CompactTextString(m) } +func (*SegmentReferenceLock) ProtoMessage() {} +func (*SegmentReferenceLock) Descriptor() ([]byte, []int) { + return fileDescriptor_82cd95f524594f49, []int{64} +} + +func (m *SegmentReferenceLock) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SegmentReferenceLock.Unmarshal(m, b) +} +func (m *SegmentReferenceLock) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SegmentReferenceLock.Marshal(b, m, deterministic) +} +func (m *SegmentReferenceLock) XXX_Merge(src proto.Message) { + xxx_messageInfo_SegmentReferenceLock.Merge(m, src) +} +func (m *SegmentReferenceLock) XXX_Size() int { + return xxx_messageInfo_SegmentReferenceLock.Size(m) +} +func (m *SegmentReferenceLock) XXX_DiscardUnknown() { + xxx_messageInfo_SegmentReferenceLock.DiscardUnknown(m) +} + +var xxx_messageInfo_SegmentReferenceLock proto.InternalMessageInfo + +func (m *SegmentReferenceLock) GetTaskID() int64 { + if m != nil { + return m.TaskID + } + return 0 +} + +func (m *SegmentReferenceLock) GetNodeID() int64 { + if m != nil { + return m.NodeID + } + return 0 +} + +func (m *SegmentReferenceLock) GetSegmentIDs() []int64 { + if m != nil { + return m.SegmentIDs + } + return nil +} + func init() { proto.RegisterEnum("milvus.proto.data.ChannelWatchState", ChannelWatchState_name, ChannelWatchState_value) proto.RegisterEnum("milvus.proto.data.CompactionType", CompactionType_name, CompactionType_value) @@ -4071,239 +4142,242 @@ func init() { proto.RegisterType((*ResendSegmentStatsRequest)(nil), "milvus.proto.data.ResendSegmentStatsRequest") proto.RegisterType((*ResendSegmentStatsResponse)(nil), "milvus.proto.data.ResendSegmentStatsResponse") proto.RegisterType((*AddSegmentRequest)(nil), "milvus.proto.data.AddSegmentRequest") + proto.RegisterType((*SegmentReferenceLock)(nil), "milvus.proto.data.SegmentReferenceLock") } func init() { proto.RegisterFile("data_coord.proto", fileDescriptor_82cd95f524594f49) } var fileDescriptor_82cd95f524594f49 = []byte{ - // 3628 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3b, 0x5d, 0x6f, 0x1b, 0xd7, - 0x95, 0x1e, 0x7e, 0x89, 0x3c, 0xfc, 0x10, 0x75, 0xed, 0xc8, 0x34, 0x6d, 0xcb, 0xf2, 0x38, 0x76, - 0x14, 0xc7, 0xb1, 0x13, 0x79, 0x83, 0x04, 0x9b, 0x2f, 0xd8, 0x96, 0xad, 0x10, 0x2b, 0x79, 0xe5, - 0x91, 0x12, 0x2f, 0x36, 0x8b, 0x25, 0x46, 0x9c, 0x2b, 0x6a, 0x22, 0xce, 0x0c, 0x3d, 0x33, 0xb4, - 0xac, 0xbc, 0xc4, 0xd8, 0x60, 0x17, 0xc8, 0x62, 0xb1, 0x2d, 0xd0, 0x97, 0x16, 0xe8, 0x43, 0xd1, - 0xa7, 0x7e, 0xa0, 0x40, 0x81, 0xa0, 0x0f, 0x69, 0xd1, 0xf7, 0xa0, 0x7d, 0xe8, 0x0f, 0xe8, 0x0f, - 0x68, 0x5f, 0xda, 0xdf, 0x50, 0xdc, 0x8f, 0xb9, 0xf3, 0x4d, 0x8e, 0x48, 0x3b, 0x7e, 0xd3, 0x3d, - 0x3c, 0xe7, 0xdc, 0x73, 0xcf, 0x3d, 0xdf, 0x73, 0x05, 0x4d, 0x4d, 0x75, 0xd5, 0x6e, 0xcf, 0xb2, - 0x6c, 0xed, 0xfa, 0xd0, 0xb6, 0x5c, 0x0b, 0x2d, 0x18, 0xfa, 0xe0, 0xf1, 0xc8, 0x61, 0xab, 0xeb, - 0xe4, 0xe7, 0x76, 0xad, 0x67, 0x19, 0x86, 0x65, 0x32, 0x50, 0xbb, 0xa1, 0x9b, 0x2e, 0xb6, 0x4d, - 0x75, 0xc0, 0xd7, 0xb5, 0x20, 0x41, 0xbb, 0xe6, 0xf4, 0xf6, 0xb1, 0xa1, 0xb2, 0x95, 0x3c, 0x07, - 0xc5, 0xbb, 0xc6, 0xd0, 0x3d, 0x92, 0x7f, 0x28, 0x41, 0xed, 0xde, 0x60, 0xe4, 0xec, 0x2b, 0xf8, - 0xd1, 0x08, 0x3b, 0x2e, 0x7a, 0x03, 0x0a, 0xbb, 0xaa, 0x83, 0x5b, 0xd2, 0xb2, 0xb4, 0x52, 0x5d, - 0x3d, 0x77, 0x3d, 0xb4, 0x2b, 0xdf, 0x6f, 0xd3, 0xe9, 0xdf, 0x56, 0x1d, 0xac, 0x50, 0x4c, 0x84, - 0xa0, 0xa0, 0xed, 0x76, 0xd6, 0x5a, 0xb9, 0x65, 0x69, 0x25, 0xaf, 0xd0, 0xbf, 0xd1, 0x12, 0x80, - 0x83, 0xfb, 0x06, 0x36, 0xdd, 0xce, 0x9a, 0xd3, 0xca, 0x2f, 0xe7, 0x57, 0xf2, 0x4a, 0x00, 0x82, - 0x64, 0xa8, 0xf5, 0xac, 0xc1, 0x00, 0xf7, 0x5c, 0xdd, 0x32, 0x3b, 0x6b, 0xad, 0x02, 0xa5, 0x0d, - 0xc1, 0xe4, 0x1f, 0x4b, 0x50, 0xe7, 0xa2, 0x39, 0x43, 0xcb, 0x74, 0x30, 0xba, 0x09, 0x25, 0xc7, - 0x55, 0xdd, 0x91, 0xc3, 0xa5, 0x3b, 0x9b, 0x28, 0xdd, 0x36, 0x45, 0x51, 0x38, 0x6a, 0xa2, 0x78, - 0xd1, 0xed, 0xf3, 0xf1, 0xed, 0x23, 0x47, 0x28, 0x44, 0x8f, 0x20, 0xff, 0x4a, 0x82, 0xe6, 0xb6, - 0xb7, 0xf4, 0xb4, 0x77, 0x0a, 0x8a, 0x3d, 0x6b, 0x64, 0xba, 0x54, 0xc0, 0xba, 0xc2, 0x16, 0xe8, - 0x22, 0xd4, 0x7a, 0xfb, 0xaa, 0x69, 0xe2, 0x41, 0xd7, 0x54, 0x0d, 0x4c, 0x45, 0xa9, 0x28, 0x55, - 0x0e, 0xbb, 0xaf, 0x1a, 0x38, 0x93, 0x44, 0xcb, 0x50, 0x1d, 0xaa, 0xb6, 0xab, 0x87, 0x74, 0x16, - 0x04, 0xa1, 0x36, 0x94, 0x75, 0xa7, 0x63, 0x0c, 0x2d, 0xdb, 0x6d, 0x15, 0x97, 0xa5, 0x95, 0xb2, - 0x22, 0xd6, 0xf2, 0x4f, 0x24, 0x58, 0xbc, 0xe5, 0x38, 0x7a, 0xdf, 0x8c, 0x49, 0xbd, 0x08, 0x25, - 0xd3, 0xd2, 0x70, 0x67, 0x8d, 0x8a, 0x9d, 0x57, 0xf8, 0x0a, 0x9d, 0x85, 0xca, 0x10, 0x63, 0xbb, - 0x6b, 0x5b, 0x03, 0x4f, 0xe8, 0x32, 0x01, 0x28, 0xd6, 0x00, 0xa3, 0x07, 0xb0, 0xe0, 0x44, 0x18, - 0xb1, 0x9b, 0xae, 0xae, 0x5e, 0xba, 0x1e, 0xb3, 0xd5, 0xeb, 0xd1, 0x4d, 0x95, 0x38, 0xb5, 0xfc, - 0x34, 0x07, 0x27, 0x05, 0x1e, 0x93, 0x95, 0xfc, 0x4d, 0xb4, 0xea, 0xe0, 0xbe, 0x10, 0x8f, 0x2d, - 0xb2, 0x68, 0x55, 0x5c, 0x47, 0x3e, 0x78, 0x1d, 0x19, 0x8c, 0x2f, 0xaa, 0xeb, 0x62, 0x5c, 0xd7, - 0x17, 0xa0, 0x8a, 0x9f, 0x0c, 0x75, 0x1b, 0x77, 0x5d, 0xdd, 0xc0, 0xad, 0xd2, 0xb2, 0xb4, 0x52, - 0x50, 0x80, 0x81, 0x76, 0x74, 0x23, 0x68, 0xad, 0x73, 0x99, 0xad, 0x55, 0xfe, 0xa9, 0x04, 0xa7, - 0x63, 0xb7, 0xc4, 0xcd, 0x5f, 0x81, 0x26, 0x3d, 0xb9, 0xaf, 0x19, 0xe2, 0x08, 0x44, 0xe1, 0x57, - 0xc6, 0x29, 0xdc, 0x47, 0x57, 0x62, 0xf4, 0x01, 0x21, 0x73, 0xd9, 0x85, 0x3c, 0x80, 0xd3, 0xeb, - 0xd8, 0xe5, 0x1b, 0x90, 0xdf, 0xb0, 0x33, 0x7d, 0xf8, 0x08, 0xfb, 0x59, 0x2e, 0xe6, 0x67, 0xbf, - 0xce, 0x09, 0x3f, 0xa3, 0x5b, 0x75, 0xcc, 0x3d, 0x0b, 0x9d, 0x83, 0x8a, 0x40, 0xe1, 0x56, 0xe1, - 0x03, 0xd0, 0xdb, 0x50, 0x24, 0x92, 0x32, 0x93, 0x68, 0xac, 0x5e, 0x4c, 0x3e, 0x53, 0x80, 0xa7, - 0xc2, 0xf0, 0x51, 0x07, 0x1a, 0x8e, 0xab, 0xda, 0x6e, 0x77, 0x68, 0x39, 0xf4, 0x9e, 0xa9, 0xe1, - 0x54, 0x57, 0xe5, 0x30, 0x07, 0x11, 0x68, 0x37, 0x9d, 0xfe, 0x16, 0xc7, 0x54, 0xea, 0x94, 0xd2, - 0x5b, 0xa2, 0xbb, 0x50, 0xc3, 0xa6, 0xe6, 0x33, 0x2a, 0x64, 0x66, 0x54, 0xc5, 0xa6, 0x26, 0xd8, - 0xf8, 0xf7, 0x53, 0xcc, 0x7e, 0x3f, 0xff, 0x27, 0x41, 0x2b, 0x7e, 0x41, 0xb3, 0x04, 0xd1, 0x77, - 0x19, 0x11, 0x66, 0x17, 0x34, 0xd6, 0xc3, 0xc5, 0x25, 0x29, 0x9c, 0x44, 0xd6, 0xe1, 0x25, 0x5f, - 0x1a, 0xfa, 0xcb, 0x73, 0x33, 0x96, 0x2f, 0x25, 0x58, 0x8c, 0xee, 0x35, 0xcb, 0xb9, 0xff, 0x09, - 0x8a, 0xba, 0xb9, 0x67, 0x79, 0xc7, 0x5e, 0x1a, 0xe3, 0x67, 0x64, 0x2f, 0x86, 0x2c, 0x1b, 0x70, - 0x76, 0x1d, 0xbb, 0x1d, 0xd3, 0xc1, 0xb6, 0x7b, 0x5b, 0x37, 0x07, 0x56, 0x7f, 0x4b, 0x75, 0xf7, - 0x67, 0xf0, 0x91, 0x90, 0xb9, 0xe7, 0x22, 0xe6, 0x2e, 0xff, 0x4c, 0x82, 0x73, 0xc9, 0xfb, 0xf1, - 0xa3, 0xb7, 0xa1, 0xbc, 0xa7, 0xe3, 0x81, 0x46, 0x74, 0x26, 0x51, 0x9d, 0x89, 0x35, 0xf1, 0x95, - 0x21, 0x41, 0xe6, 0x27, 0xbc, 0x98, 0x62, 0xa0, 0xdb, 0xae, 0xad, 0x9b, 0xfd, 0x0d, 0xdd, 0x71, - 0x15, 0x86, 0x1f, 0xd0, 0x67, 0x3e, 0xbb, 0x65, 0xfe, 0xaf, 0x04, 0x4b, 0xeb, 0xd8, 0xbd, 0x23, - 0x42, 0x2d, 0xf9, 0x5d, 0x77, 0x5c, 0xbd, 0xe7, 0x3c, 0xdb, 0x02, 0x24, 0x43, 0x3e, 0x95, 0xbf, - 0x27, 0xc1, 0x85, 0x54, 0x61, 0xb8, 0xea, 0x78, 0x28, 0xf1, 0x02, 0x6d, 0x72, 0x28, 0xf9, 0x17, - 0x7c, 0xf4, 0x89, 0x3a, 0x18, 0xe1, 0x2d, 0x55, 0xb7, 0x59, 0x28, 0x99, 0x32, 0xb0, 0xfe, 0x52, - 0x82, 0xf3, 0xeb, 0xd8, 0xdd, 0xf2, 0xd2, 0xcc, 0x0b, 0xd4, 0xce, 0xe4, 0x6a, 0x43, 0xfe, 0x7f, - 0x76, 0x99, 0x89, 0xd2, 0xbe, 0x10, 0xf5, 0x2d, 0x51, 0x3f, 0x08, 0x38, 0xe4, 0x1d, 0x56, 0x0b, - 0x70, 0xe5, 0xc9, 0xff, 0x2d, 0xc1, 0x99, 0x5b, 0xbd, 0x47, 0x23, 0xdd, 0xc6, 0x1c, 0x69, 0xc3, - 0xea, 0x1d, 0x4c, 0xaf, 0x5a, 0xbf, 0x6e, 0xca, 0x85, 0xea, 0xa6, 0x09, 0xd5, 0x2f, 0x95, 0x43, - 0xc1, 0x03, 0xac, 0x3a, 0x2f, 0x56, 0x8e, 0xa7, 0x79, 0xa8, 0x7d, 0xc2, 0xeb, 0x25, 0x9a, 0x56, - 0xa3, 0x76, 0x21, 0x25, 0xdb, 0x45, 0xa0, 0xc4, 0x4a, 0xaa, 0xba, 0xd6, 0xa1, 0xee, 0x60, 0x7c, - 0x30, 0x4d, 0x12, 0xad, 0x11, 0x42, 0x91, 0xfc, 0x36, 0x60, 0x61, 0x64, 0xee, 0x91, 0x16, 0x00, - 0x6b, 0x5c, 0x51, 0xac, 0x12, 0x9f, 0x1c, 0x89, 0xe3, 0x84, 0xe8, 0x23, 0x98, 0x8f, 0xf2, 0x2a, - 0x66, 0xe2, 0x15, 0x25, 0x43, 0x1d, 0x68, 0x6a, 0xb6, 0x35, 0x1c, 0x62, 0xad, 0xeb, 0x78, 0xac, - 0x4a, 0xd9, 0x58, 0x71, 0x3a, 0x8f, 0x95, 0xfc, 0x95, 0x04, 0x8b, 0x0f, 0x55, 0xb7, 0xb7, 0xbf, - 0x66, 0x70, 0x63, 0x9d, 0xc1, 0xd5, 0xdf, 0x87, 0xca, 0x63, 0x7e, 0x11, 0x5e, 0x3c, 0xbf, 0x90, - 0x20, 0x50, 0xf0, 0xca, 0x15, 0x9f, 0x42, 0xfe, 0x56, 0x82, 0x53, 0xb4, 0xe1, 0xf2, 0xa4, 0xfb, - 0xee, 0x83, 0xce, 0x84, 0xa6, 0x0b, 0x5d, 0x81, 0x86, 0xa1, 0xda, 0x07, 0xdb, 0x3e, 0x4e, 0x91, - 0xe2, 0x44, 0xa0, 0xf2, 0x13, 0x00, 0xbe, 0xda, 0x74, 0xfa, 0x53, 0xc8, 0xff, 0x0e, 0xcc, 0xf1, - 0x5d, 0x79, 0xfc, 0x99, 0x74, 0xb1, 0x1e, 0xba, 0xfc, 0x07, 0x09, 0x1a, 0x7e, 0x46, 0xa1, 0x5e, - 0xd5, 0x80, 0x9c, 0xf0, 0xa5, 0x5c, 0x67, 0x0d, 0xbd, 0x0f, 0x25, 0xd6, 0x8c, 0x73, 0xde, 0x97, - 0xc3, 0xbc, 0x79, 0xa3, 0x1e, 0x48, 0x4b, 0x14, 0xa0, 0x70, 0x22, 0xa2, 0x23, 0x11, 0x85, 0x85, - 0x57, 0xfb, 0x10, 0xd4, 0x81, 0xf9, 0x70, 0x11, 0xeb, 0xf9, 0xcc, 0x72, 0x5a, 0xf4, 0x5d, 0x53, - 0x5d, 0x95, 0x06, 0xdf, 0x46, 0xa8, 0x86, 0x75, 0xe4, 0xbf, 0x17, 0xa1, 0x1a, 0x38, 0x65, 0xec, - 0x24, 0xd1, 0x2b, 0xcd, 0x4d, 0xce, 0x23, 0xf9, 0x78, 0x27, 0x75, 0x19, 0x1a, 0x3a, 0xad, 0x5d, - 0xba, 0xdc, 0x14, 0x69, 0xb2, 0xa9, 0x28, 0x75, 0x06, 0xe5, 0x7e, 0x81, 0x96, 0xa0, 0x6a, 0x8e, - 0x8c, 0xae, 0xb5, 0xd7, 0xb5, 0xad, 0x43, 0x87, 0xb7, 0x64, 0x15, 0x73, 0x64, 0xfc, 0xeb, 0x9e, - 0x62, 0x1d, 0x3a, 0x7e, 0xd5, 0x5f, 0x3a, 0x66, 0xd5, 0xbf, 0x04, 0x55, 0x43, 0x7d, 0x42, 0xb8, - 0x76, 0xcd, 0x91, 0x41, 0xbb, 0xb5, 0xbc, 0x52, 0x31, 0xd4, 0x27, 0x8a, 0x75, 0x78, 0x7f, 0x64, - 0xa0, 0x15, 0x68, 0x0e, 0x54, 0xc7, 0xed, 0x06, 0xdb, 0xbd, 0x32, 0x6d, 0xf7, 0x1a, 0x04, 0x7e, - 0xd7, 0x6f, 0xf9, 0xe2, 0xfd, 0x43, 0x65, 0x86, 0xfe, 0x41, 0x33, 0x06, 0x3e, 0x23, 0xc8, 0xde, - 0x3f, 0x68, 0xc6, 0x40, 0xb0, 0x79, 0x07, 0xe6, 0x76, 0x69, 0x45, 0xe8, 0xb4, 0xaa, 0xa9, 0x11, - 0xea, 0x1e, 0x29, 0x06, 0x59, 0xe1, 0xa8, 0x78, 0xe8, 0xe8, 0x3d, 0xa8, 0xd0, 0x54, 0x4c, 0x69, - 0x6b, 0x99, 0x68, 0x7d, 0x02, 0x42, 0xad, 0xe1, 0x81, 0xab, 0x52, 0xea, 0x7a, 0x36, 0x6a, 0x41, - 0x80, 0xde, 0x80, 0x93, 0x3d, 0x1b, 0xab, 0x2e, 0xd6, 0x6e, 0x1f, 0xdd, 0xb1, 0x8c, 0xa1, 0x4a, - 0x8d, 0xa9, 0xd5, 0xa0, 0x23, 0x8d, 0xa4, 0x9f, 0x48, 0x60, 0xe8, 0x89, 0xd5, 0x3d, 0xdb, 0x32, - 0x5a, 0xf3, 0x2c, 0x30, 0x84, 0xa1, 0xe8, 0x3c, 0x80, 0x17, 0xba, 0x55, 0xb7, 0xd5, 0xa4, 0xb7, - 0x58, 0xe1, 0x90, 0x5b, 0xae, 0xfc, 0x05, 0x9c, 0xf2, 0x2d, 0x24, 0x70, 0x1b, 0xf1, 0x8b, 0x95, - 0xa6, 0xbd, 0xd8, 0xf1, 0xb5, 0xfc, 0x9f, 0x0a, 0xb0, 0xb8, 0xad, 0x3e, 0xc6, 0xcf, 0xbf, 0x6d, - 0xc8, 0x14, 0x8f, 0x37, 0x60, 0x81, 0x76, 0x0a, 0xab, 0x01, 0x79, 0xc6, 0x64, 0xe0, 0xe0, 0x75, - 0xc6, 0x09, 0xd1, 0x87, 0xa4, 0x74, 0xc0, 0xbd, 0x83, 0x2d, 0x4b, 0xf7, 0xb3, 0xef, 0xf9, 0x04, - 0x3e, 0x77, 0x04, 0x96, 0x12, 0xa4, 0x40, 0x5b, 0xf1, 0xd0, 0xc6, 0xf2, 0xee, 0x2b, 0x63, 0xfb, - 0x51, 0x5f, 0xfb, 0xd1, 0x08, 0x87, 0x5a, 0x30, 0xc7, 0xb3, 0x3b, 0xf5, 0xfb, 0xb2, 0xe2, 0x2d, - 0xd1, 0x16, 0x9c, 0x64, 0x27, 0xd8, 0xe6, 0x46, 0xcd, 0x0e, 0x5f, 0xce, 0x74, 0xf8, 0x24, 0xd2, - 0xb0, 0x4f, 0x54, 0x8e, 0xeb, 0x13, 0x2d, 0x98, 0xe3, 0x76, 0x4a, 0x63, 0x41, 0x59, 0xf1, 0x96, - 0xe4, 0x9a, 0x75, 0x3a, 0xe3, 0xd3, 0xcd, 0x7e, 0xab, 0x4a, 0x7f, 0xf3, 0x01, 0xa4, 0xe5, 0x02, - 0x5f, 0x9f, 0x13, 0x26, 0x27, 0x1f, 0x40, 0x59, 0x58, 0x78, 0x2e, 0xb3, 0x85, 0x0b, 0x9a, 0x68, - 0x8c, 0xce, 0x47, 0x62, 0xb4, 0xfc, 0x47, 0x09, 0x6a, 0x6b, 0xe4, 0x48, 0x1b, 0x56, 0x9f, 0x66, - 0x94, 0xcb, 0xd0, 0xb0, 0x71, 0xcf, 0xb2, 0xb5, 0x2e, 0x36, 0x5d, 0x5b, 0xc7, 0xac, 0x3b, 0x2f, - 0x28, 0x75, 0x06, 0xbd, 0xcb, 0x80, 0x04, 0x8d, 0x84, 0x5d, 0xc7, 0x55, 0x8d, 0x61, 0x77, 0x8f, - 0xb8, 0x77, 0x8e, 0xa1, 0x09, 0x28, 0xf5, 0xee, 0x8b, 0x50, 0xf3, 0xd1, 0x5c, 0x8b, 0xee, 0x5f, - 0x50, 0xaa, 0x02, 0xb6, 0x63, 0xa1, 0x97, 0xa1, 0x41, 0x75, 0xda, 0x1d, 0x58, 0xfd, 0x2e, 0xe9, - 0x64, 0x79, 0xb2, 0xa9, 0x69, 0x5c, 0x2c, 0x72, 0x57, 0x61, 0x2c, 0x47, 0xff, 0x1c, 0xf3, 0x74, - 0x23, 0xb0, 0xb6, 0xf5, 0xcf, 0x31, 0xc9, 0xf5, 0x75, 0x92, 0x3b, 0xef, 0x5b, 0x1a, 0xde, 0x99, - 0xb2, 0xd2, 0xc8, 0x30, 0xc5, 0x3c, 0x07, 0x15, 0x71, 0x02, 0x7e, 0x24, 0x1f, 0x80, 0xee, 0x41, - 0xc3, 0x2b, 0x42, 0xbb, 0xac, 0xd7, 0x2a, 0xa4, 0x56, 0x7e, 0x81, 0xec, 0xe7, 0x28, 0x75, 0x8f, - 0x8c, 0x2e, 0xe5, 0x7b, 0x50, 0x0b, 0xfe, 0x4c, 0x76, 0xdd, 0x8e, 0x1a, 0x8a, 0x00, 0x10, 0x6b, - 0xbc, 0x3f, 0x32, 0xc8, 0x9d, 0xf2, 0xc0, 0xe2, 0x2d, 0xe5, 0x2f, 0x25, 0xa8, 0xf3, 0x94, 0xbd, - 0x2d, 0x26, 0xf0, 0xf4, 0x68, 0x12, 0x3d, 0x1a, 0xfd, 0x1b, 0xfd, 0x73, 0x78, 0x44, 0xf7, 0x72, - 0x62, 0x10, 0xa0, 0x4c, 0x68, 0x75, 0x1c, 0xca, 0xd7, 0x59, 0x7a, 0xfb, 0xa7, 0xc4, 0xd0, 0xf8, - 0xd5, 0x50, 0x43, 0x6b, 0xc1, 0x9c, 0xaa, 0x69, 0x36, 0x76, 0x1c, 0x2e, 0x87, 0xb7, 0x24, 0xbf, - 0x3c, 0xc6, 0xb6, 0xe3, 0x99, 0x7c, 0x5e, 0xf1, 0x96, 0xe8, 0x3d, 0x28, 0x8b, 0x72, 0x3a, 0x9f, - 0x54, 0x42, 0x05, 0xe5, 0xe4, 0xbd, 0xa8, 0xa0, 0x90, 0x7f, 0x93, 0x83, 0x06, 0x57, 0xd8, 0x6d, - 0x9e, 0x53, 0xc7, 0x3b, 0xdf, 0x6d, 0xa8, 0xed, 0xf9, 0xbe, 0x3f, 0x6e, 0xe6, 0x14, 0x0c, 0x11, - 0x21, 0x9a, 0x49, 0x0e, 0x18, 0xce, 0xea, 0x85, 0x99, 0xb2, 0x7a, 0xf1, 0xb8, 0x11, 0x2c, 0x5e, - 0xe7, 0x95, 0x12, 0xea, 0x3c, 0xf9, 0x3f, 0xa0, 0x1a, 0x60, 0x40, 0x23, 0x34, 0x1b, 0x56, 0x71, - 0x8d, 0x79, 0x4b, 0x74, 0xd3, 0xaf, 0x6d, 0x98, 0xaa, 0xce, 0x24, 0xc8, 0x12, 0x29, 0x6b, 0xe4, - 0x9f, 0x4b, 0x50, 0xe2, 0x9c, 0x2f, 0x40, 0x95, 0x07, 0x1d, 0x5a, 0xf7, 0x31, 0xee, 0xc0, 0x41, - 0xa4, 0xf0, 0x7b, 0x76, 0x51, 0xe7, 0x0c, 0x94, 0x23, 0xf1, 0x66, 0x8e, 0xa7, 0x05, 0xef, 0xa7, - 0x40, 0x90, 0x21, 0x3f, 0xd1, 0xf8, 0xf2, 0xad, 0x44, 0x07, 0xed, 0x0a, 0xee, 0x59, 0x8f, 0xb1, - 0x7d, 0x34, 0xfb, 0x38, 0xf3, 0xdd, 0x80, 0x41, 0x67, 0xec, 0x0f, 0x05, 0x01, 0x7a, 0xd7, 0x57, - 0x77, 0x3e, 0x69, 0x9a, 0x13, 0x8c, 0x30, 0xdc, 0x1c, 0x7d, 0xb5, 0x7f, 0x9f, 0x0d, 0x66, 0xc3, - 0x47, 0x99, 0xb6, 0xae, 0x79, 0x26, 0x6d, 0x87, 0xfc, 0x03, 0x09, 0xce, 0xac, 0x63, 0xf7, 0x5e, - 0xb8, 0xb9, 0x7f, 0xd1, 0x52, 0x19, 0xd0, 0x4e, 0x12, 0x6a, 0x96, 0x5b, 0x6f, 0x43, 0x59, 0x8c, - 0x29, 0xd8, 0xc8, 0x5c, 0xac, 0xe5, 0xff, 0x91, 0xa0, 0xc5, 0x77, 0xa1, 0x7b, 0x92, 0x92, 0x7a, - 0x80, 0x5d, 0xac, 0x7d, 0xd7, 0x7d, 0xf3, 0xef, 0x25, 0x68, 0x06, 0x23, 0x3e, 0x0d, 0xda, 0x6f, - 0x41, 0x91, 0x8e, 0x27, 0xb8, 0x04, 0x13, 0x8d, 0x95, 0x61, 0x93, 0x90, 0x41, 0xcb, 0xbc, 0x1d, - 0x91, 0x9c, 0xf8, 0xd2, 0x4f, 0x3b, 0xf9, 0xe3, 0xa7, 0x1d, 0x9e, 0x86, 0xad, 0x11, 0xe1, 0xcb, - 0xc6, 0xa1, 0x3e, 0x40, 0xfe, 0x3a, 0x07, 0x2d, 0xbf, 0x1f, 0xf9, 0xce, 0xe3, 0x7e, 0x4a, 0xb5, - 0x9a, 0x7f, 0x46, 0xd5, 0x6a, 0x61, 0xf6, 0x58, 0x5f, 0x4c, 0x8a, 0xf5, 0xbf, 0xcb, 0x41, 0xc3, - 0xd7, 0xda, 0xd6, 0x40, 0x35, 0xd1, 0x22, 0x94, 0x86, 0x03, 0xd5, 0x9f, 0x3e, 0xf2, 0x15, 0xda, - 0x16, 0x75, 0x4e, 0x58, 0x4f, 0xaf, 0x25, 0xdd, 0x61, 0xca, 0x45, 0x28, 0x11, 0x16, 0xa4, 0x1d, - 0x64, 0x0d, 0x05, 0x6d, 0xea, 0x79, 0x6d, 0xc5, 0x8c, 0x85, 0xf4, 0xf3, 0xd7, 0x00, 0xf1, 0x1b, - 0xee, 0xea, 0x66, 0xd7, 0xc1, 0x3d, 0xcb, 0xd4, 0xd8, 0xdd, 0x17, 0x95, 0x26, 0xff, 0xa5, 0x63, - 0x6e, 0x33, 0x38, 0x7a, 0x0b, 0x0a, 0xee, 0xd1, 0x90, 0x45, 0xf1, 0x46, 0x62, 0x74, 0xf4, 0xe5, - 0xda, 0x39, 0x1a, 0x62, 0x85, 0xa2, 0xa3, 0x25, 0x00, 0xc2, 0xca, 0xb5, 0xd5, 0xc7, 0x3c, 0x25, - 0x16, 0x94, 0x00, 0x84, 0x58, 0xb3, 0xa7, 0xc3, 0x39, 0x96, 0x3a, 0xf8, 0x52, 0xfe, 0x26, 0x07, - 0x4d, 0x9f, 0xa5, 0x82, 0x9d, 0xd1, 0xc0, 0x4d, 0xd5, 0xdf, 0xf8, 0x66, 0x70, 0x52, 0xdd, 0xf0, - 0x21, 0x54, 0xf9, 0x7d, 0x1e, 0xc3, 0x1e, 0x80, 0x91, 0x6c, 0x8c, 0x31, 0xd0, 0xe2, 0x33, 0x32, - 0xd0, 0xd2, 0x31, 0x0d, 0x54, 0xde, 0x86, 0x45, 0x2f, 0xee, 0xf9, 0x08, 0x9b, 0xd8, 0x55, 0xc7, - 0x14, 0x1c, 0x17, 0xa0, 0xca, 0xf2, 0x19, 0x4b, 0xe4, 0xac, 0x54, 0x87, 0x5d, 0xd1, 0xe1, 0xca, - 0xff, 0x09, 0xa7, 0x68, 0xdc, 0x88, 0x8e, 0x72, 0xb3, 0xcc, 0xd5, 0x65, 0xd1, 0x08, 0x90, 0xa2, - 0x9f, 0x59, 0x77, 0x45, 0x09, 0xc1, 0xe4, 0x0d, 0x78, 0x29, 0xc2, 0x7f, 0x86, 0xbc, 0x40, 0x4a, - 0xa1, 0xc5, 0xed, 0xf0, 0x67, 0xe2, 0xe9, 0xb3, 0xdf, 0x79, 0x31, 0xb9, 0xed, 0xea, 0x5a, 0xd4, - 0xbe, 0x34, 0xf4, 0x01, 0x54, 0x4c, 0x7c, 0xd8, 0x0d, 0x06, 0xdf, 0x0c, 0x03, 0xba, 0xb2, 0x89, - 0x0f, 0xe9, 0x5f, 0xf2, 0x7d, 0x38, 0x1d, 0x13, 0x75, 0x96, 0xb3, 0xff, 0x56, 0x82, 0x33, 0x6b, - 0xb6, 0x35, 0xfc, 0x44, 0xb7, 0xdd, 0x91, 0x3a, 0x08, 0x7f, 0x28, 0x7a, 0x3e, 0x6d, 0xdc, 0x47, - 0x81, 0x34, 0xcc, 0xe2, 0xf2, 0xb5, 0x04, 0x73, 0x8d, 0x0b, 0xc5, 0x0f, 0x1d, 0x48, 0xda, 0x7f, - 0xc9, 0x27, 0x09, 0xcf, 0xf1, 0x26, 0x24, 0x9b, 0x2c, 0x55, 0x4a, 0xe2, 0xd4, 0x27, 0x3f, 0xed, - 0xd4, 0x27, 0xc5, 0xf3, 0x0b, 0xcf, 0xc8, 0xf3, 0x8f, 0xdd, 0x86, 0x7c, 0x04, 0xe1, 0x89, 0x1c, - 0x0d, 0xb9, 0x53, 0x8d, 0xf2, 0x6e, 0x03, 0xf8, 0xd3, 0x29, 0xfe, 0xca, 0x27, 0x0b, 0x9b, 0x00, - 0x15, 0xb9, 0x2d, 0x11, 0x65, 0xe9, 0x54, 0x39, 0x34, 0x2f, 0x79, 0x00, 0xed, 0x24, 0x2b, 0x9d, - 0xc5, 0xf2, 0xbf, 0xce, 0x01, 0xb0, 0x27, 0x61, 0x3b, 0xaa, 0x73, 0x30, 0x5d, 0x45, 0x79, 0x09, - 0xea, 0xbe, 0xc1, 0xf8, 0xfe, 0x1e, 0xb4, 0x22, 0x8d, 0xb8, 0x84, 0x28, 0x6c, 0x09, 0x4e, 0xac, - 0xd8, 0xd5, 0x28, 0x9f, 0x80, 0xd7, 0x30, 0xa3, 0x88, 0x04, 0x3d, 0x74, 0x16, 0x2a, 0xb6, 0x75, - 0xd8, 0x25, 0x6e, 0xa6, 0x79, 0xaf, 0xda, 0x6c, 0xeb, 0x90, 0x38, 0x9f, 0x86, 0x4e, 0xc3, 0x9c, - 0xab, 0x3a, 0x07, 0x84, 0x7f, 0x89, 0xa5, 0x3b, 0xb2, 0xec, 0x68, 0xe8, 0x14, 0x14, 0xf7, 0xf4, - 0x01, 0x76, 0x5a, 0x73, 0x94, 0x25, 0x5b, 0xa0, 0xb7, 0xbd, 0xf7, 0x1c, 0xe5, 0xcc, 0xdf, 0xa3, - 0xd9, 0x93, 0x8e, 0x6f, 0x25, 0x98, 0xf7, 0xb5, 0x46, 0x03, 0x10, 0x89, 0x69, 0x34, 0x9e, 0xdd, - 0xb1, 0x34, 0x16, 0x2a, 0x1a, 0x29, 0x9f, 0x58, 0x18, 0x21, 0x8b, 0x5a, 0x3e, 0xc9, 0xb8, 0xba, - 0x9c, 0x9c, 0x8b, 0x1c, 0x5a, 0xd7, 0xbc, 0x2f, 0x3c, 0x25, 0xdb, 0x3a, 0xec, 0x68, 0x42, 0x1b, - 0xec, 0x59, 0x1b, 0xab, 0x42, 0x89, 0x36, 0xee, 0xd0, 0x97, 0x6d, 0x97, 0xa0, 0x8e, 0x6d, 0xdb, - 0xb2, 0xbb, 0x06, 0x76, 0x1c, 0xb5, 0x8f, 0x79, 0xd1, 0x55, 0xa3, 0xc0, 0x4d, 0x06, 0x93, 0xbf, - 0xc9, 0x43, 0xc3, 0x3f, 0x8a, 0xf7, 0x5d, 0x47, 0xd7, 0xbc, 0xef, 0x3a, 0xba, 0x46, 0x82, 0xb9, - 0xcd, 0x42, 0x61, 0x20, 0x98, 0x73, 0x48, 0x47, 0x23, 0x79, 0x90, 0x38, 0x98, 0x69, 0x69, 0xd8, - 0xbf, 0x58, 0xf0, 0x40, 0xfc, 0x5e, 0x43, 0xf6, 0x51, 0xc8, 0x60, 0x1f, 0xc5, 0x0c, 0xf6, 0x51, - 0x4a, 0xb0, 0x8f, 0x45, 0x28, 0xed, 0x8e, 0x7a, 0x07, 0xd8, 0xe5, 0xe5, 0x11, 0x5f, 0x85, 0xed, - 0xa6, 0x1c, 0xb1, 0x1b, 0x61, 0x1e, 0x95, 0xa0, 0x79, 0x9c, 0x85, 0x0a, 0xfb, 0xb8, 0xd0, 0x75, - 0x1d, 0x3a, 0x65, 0xcd, 0x2b, 0x65, 0x06, 0xd8, 0x71, 0xd0, 0x3b, 0x5e, 0xef, 0x50, 0x4d, 0x72, - 0x74, 0x1a, 0x71, 0x22, 0x16, 0xe2, 0x75, 0x0e, 0x97, 0xa1, 0x41, 0x9f, 0xf3, 0x3e, 0x1a, 0x61, - 0xfb, 0x48, 0xdd, 0x1d, 0xe0, 0x56, 0x8d, 0x8a, 0x53, 0x27, 0xd0, 0x07, 0x1e, 0x90, 0x28, 0x84, - 0xa2, 0xe9, 0xa6, 0x86, 0x9f, 0x60, 0xad, 0x55, 0xa7, 0x48, 0x54, 0xd5, 0x1d, 0x06, 0x92, 0x3f, - 0x03, 0xe4, 0xef, 0x31, 0x5b, 0x57, 0x18, 0xb9, 0xc4, 0x5c, 0xf4, 0x12, 0xe5, 0x5f, 0x48, 0xb0, - 0x10, 0xdc, 0x6c, 0xda, 0xd4, 0xf8, 0x01, 0x54, 0xd9, 0x34, 0xba, 0x4b, 0x5c, 0x93, 0xf7, 0x85, - 0xe7, 0xc7, 0x6a, 0x4f, 0x01, 0xdd, 0x8f, 0x50, 0x97, 0xa0, 0x7e, 0x68, 0xd9, 0x07, 0xba, 0xd9, - 0xef, 0x12, 0xc9, 0x3c, 0x87, 0xa8, 0x71, 0xe0, 0x7d, 0x02, 0x93, 0xbf, 0x92, 0x60, 0xe9, 0xe3, - 0xa1, 0xa6, 0xba, 0x38, 0x50, 0x23, 0xcc, 0xfa, 0x74, 0xe6, 0x2d, 0xef, 0xf5, 0x4a, 0x2e, 0xdb, - 0x44, 0x95, 0x61, 0xcb, 0x9b, 0x70, 0x46, 0xc1, 0x0e, 0x36, 0xb5, 0xd0, 0x8f, 0xd3, 0x4a, 0x21, - 0x0f, 0xa1, 0x9d, 0xc4, 0x6e, 0x96, 0xbb, 0x67, 0xc5, 0x5a, 0xd7, 0x26, 0x6c, 0x5d, 0x1e, 0x7b, - 0x48, 0x8d, 0x40, 0xf7, 0x71, 0xe5, 0xbf, 0x4a, 0xb0, 0x70, 0x4b, 0xf3, 0xf6, 0x7b, 0x6e, 0x35, - 0x61, 0xb4, 0x66, 0xca, 0xc7, 0x6b, 0xa6, 0x67, 0x15, 0x48, 0x78, 0x38, 0x35, 0x47, 0x86, 0x97, - 0x26, 0x6c, 0xfa, 0x6d, 0xf7, 0xea, 0x8f, 0x24, 0x58, 0x88, 0x75, 0xfc, 0xa8, 0x01, 0xf0, 0xb1, - 0xd9, 0xe3, 0xa3, 0x90, 0xe6, 0x09, 0x54, 0x83, 0xb2, 0x37, 0x18, 0x69, 0x4a, 0xa8, 0x0a, 0x73, - 0x3b, 0x16, 0xc5, 0x6e, 0xe6, 0x50, 0x13, 0x6a, 0x8c, 0x70, 0xd4, 0xeb, 0x61, 0xc7, 0x69, 0xe6, - 0x05, 0xe4, 0x9e, 0xaa, 0x0f, 0x46, 0x36, 0x6e, 0x16, 0x50, 0x1d, 0x2a, 0x3b, 0x16, 0x7f, 0xf0, - 0xd3, 0x2c, 0x22, 0x04, 0x0d, 0xef, 0xf5, 0x0f, 0x27, 0x2a, 0x05, 0x60, 0x1e, 0xd9, 0xdc, 0xd5, - 0xbd, 0x60, 0x6f, 0x4c, 0x1a, 0x46, 0x74, 0x1a, 0x4e, 0x7e, 0x6c, 0x6a, 0x78, 0x4f, 0x37, 0xb1, - 0xe6, 0xff, 0xd4, 0x3c, 0x81, 0x4e, 0xc2, 0x7c, 0xc7, 0x34, 0xb1, 0x1d, 0x00, 0x4a, 0x04, 0xb8, - 0x89, 0xed, 0x3e, 0x0e, 0x00, 0x73, 0x68, 0x01, 0xea, 0x9b, 0xfa, 0x93, 0x00, 0x28, 0xbf, 0xfa, - 0xe7, 0xd3, 0x50, 0x59, 0x53, 0x5d, 0xf5, 0x8e, 0x65, 0xd9, 0x1a, 0x1a, 0x02, 0xa2, 0x8f, 0xe2, - 0x8c, 0xa1, 0x65, 0x8a, 0xd7, 0xa3, 0xe8, 0x8d, 0x94, 0xb2, 0x26, 0x8e, 0xca, 0xcd, 0xa5, 0x7d, - 0x25, 0x85, 0x22, 0x82, 0x2e, 0x9f, 0x40, 0x06, 0xdd, 0x91, 0x34, 0xdc, 0x3b, 0x7a, 0xef, 0xc0, - 0xfb, 0xdc, 0x3f, 0x66, 0xc7, 0x08, 0xaa, 0xb7, 0x63, 0xe4, 0x51, 0x2a, 0x5f, 0xb0, 0x97, 0x8b, - 0x9e, 0xbf, 0xc8, 0x27, 0xd0, 0x23, 0x38, 0xb5, 0x8e, 0x03, 0xf1, 0xc1, 0xdb, 0x70, 0x35, 0x7d, - 0xc3, 0x18, 0xf2, 0x31, 0xb7, 0xdc, 0x80, 0x22, 0x9d, 0xae, 0xa1, 0xa4, 0x10, 0x12, 0xfc, 0xf7, - 0x8b, 0xf6, 0x72, 0x3a, 0x82, 0xe0, 0xf6, 0x19, 0xcc, 0x47, 0x9e, 0x88, 0xa3, 0x57, 0x13, 0xc8, - 0x92, 0x1f, 0xfb, 0xb7, 0xaf, 0x66, 0x41, 0x15, 0x7b, 0xf5, 0xa1, 0x11, 0x7e, 0x52, 0x87, 0x56, - 0x12, 0xe8, 0x13, 0x9f, 0xf7, 0xb6, 0x5f, 0xcd, 0x80, 0x29, 0x36, 0x32, 0xa0, 0x19, 0x7d, 0xb2, - 0x8c, 0xae, 0x8e, 0x65, 0x10, 0x36, 0xb7, 0xd7, 0x32, 0xe1, 0x8a, 0xed, 0x8e, 0xa8, 0x11, 0xc4, - 0x9e, 0xcc, 0xa2, 0xeb, 0xc9, 0x6c, 0xd2, 0xde, 0xf2, 0xb6, 0x6f, 0x64, 0xc6, 0x17, 0x5b, 0xff, - 0x17, 0x9b, 0xea, 0x27, 0x3d, 0x3b, 0x45, 0x6f, 0x26, 0xb3, 0x1b, 0xf3, 0x5e, 0xb6, 0xbd, 0x7a, - 0x1c, 0x12, 0x21, 0xc4, 0x17, 0x74, 0x1c, 0x9f, 0xf0, 0x74, 0x33, 0xea, 0x77, 0x1e, 0xbf, 0xf4, - 0x37, 0xa9, 0xed, 0x37, 0x8f, 0x41, 0x21, 0x04, 0xb0, 0xa2, 0x8f, 0xc2, 0x3d, 0x37, 0xbc, 0x31, - 0xd1, 0x6a, 0xa6, 0xf3, 0xc1, 0x4f, 0x61, 0x3e, 0xf2, 0xb0, 0x22, 0xd1, 0x6b, 0x92, 0x1f, 0x5f, - 0xb4, 0xc7, 0xa5, 0x55, 0xe6, 0x92, 0x91, 0xaf, 0x1b, 0x28, 0xc5, 0xfa, 0x13, 0xbe, 0x80, 0xb4, - 0xaf, 0x66, 0x41, 0x15, 0x07, 0x71, 0x68, 0xb8, 0x8c, 0x7c, 0x21, 0x40, 0xd7, 0x92, 0x79, 0x24, - 0x7f, 0xdd, 0x68, 0xbf, 0x9e, 0x11, 0x5b, 0x6c, 0xda, 0x05, 0x58, 0xc7, 0xee, 0x26, 0x76, 0x6d, - 0x62, 0x23, 0x57, 0x12, 0x55, 0xee, 0x23, 0x78, 0xdb, 0xbc, 0x32, 0x11, 0x4f, 0x6c, 0xf0, 0x6f, - 0x80, 0xbc, 0x14, 0x1b, 0x78, 0xd6, 0x73, 0x69, 0xec, 0x10, 0x95, 0x4d, 0x3c, 0x27, 0xdd, 0xcd, - 0x23, 0x68, 0x6e, 0xaa, 0x26, 0x69, 0x9f, 0x7d, 0xbe, 0xd7, 0x12, 0x05, 0x8b, 0xa2, 0xa5, 0x68, - 0x2b, 0x15, 0x5b, 0x1c, 0xe6, 0x50, 0xe4, 0x50, 0x55, 0xb8, 0x20, 0x8e, 0xc6, 0x16, 0x5f, 0x1b, - 0x11, 0xc4, 0x94, 0xd8, 0x32, 0x06, 0x5f, 0x6c, 0xfc, 0x54, 0xa2, 0xff, 0x7a, 0x10, 0x41, 0x78, - 0xa8, 0xbb, 0xfb, 0x5b, 0x03, 0xd5, 0x74, 0xb2, 0x88, 0x40, 0x11, 0x8f, 0x21, 0x02, 0xc7, 0x17, - 0x22, 0x68, 0x50, 0x0f, 0xcd, 0x28, 0x51, 0xd2, 0xdb, 0x9c, 0xa4, 0x29, 0x69, 0x7b, 0x65, 0x32, - 0xa2, 0xd8, 0x65, 0x1f, 0xea, 0x9e, 0xbd, 0x32, 0xe5, 0xbe, 0x9a, 0x26, 0xa9, 0x8f, 0x93, 0xe2, - 0x6e, 0xc9, 0xa8, 0x41, 0x77, 0x8b, 0x8f, 0x60, 0x50, 0xb6, 0xd1, 0xdd, 0x38, 0x77, 0x4b, 0x9f, - 0xeb, 0xb0, 0x78, 0x12, 0x19, 0x77, 0x26, 0x07, 0xab, 0xc4, 0xe9, 0x6d, 0x62, 0x3c, 0x49, 0x99, - 0x9e, 0xca, 0x27, 0xd0, 0x43, 0x28, 0xb1, 0xce, 0x0b, 0xbd, 0x3c, 0xbe, 0x29, 0xe3, 0xdc, 0x2f, - 0x4f, 0xc0, 0x12, 0x8c, 0x0f, 0xe0, 0x74, 0x4a, 0x4b, 0x96, 0x98, 0xe7, 0xc6, 0xb7, 0x6f, 0x93, - 0xbc, 0x5c, 0x05, 0x14, 0x7f, 0xda, 0x9f, 0x78, 0x4d, 0xa9, 0xff, 0x01, 0x90, 0x61, 0x8b, 0xf8, - 0xab, 0xfd, 0xc4, 0x2d, 0x52, 0x1f, 0xf7, 0x4f, 0xda, 0xe2, 0x01, 0x80, 0xdf, 0x78, 0x25, 0xde, - 0x47, 0xac, 0x2f, 0x9b, 0xc0, 0x72, 0xf5, 0x6f, 0x25, 0x28, 0x7b, 0x2f, 0x61, 0x5e, 0x40, 0x71, - 0xff, 0x02, 0xaa, 0xed, 0x4f, 0x61, 0x3e, 0xf2, 0xa4, 0x3e, 0xd1, 0x79, 0x92, 0x9f, 0xdd, 0x4f, - 0xba, 0xa1, 0x87, 0xfc, 0x9f, 0x92, 0x45, 0xe2, 0x7d, 0x25, 0xad, 0x62, 0x8f, 0xe6, 0xdc, 0x09, - 0x8c, 0x9f, 0x7b, 0x86, 0xbd, 0x0f, 0x10, 0xc8, 0x80, 0xe3, 0x3f, 0x4f, 0x92, 0xa0, 0x3e, 0x49, - 0xe0, 0xcd, 0x63, 0xc6, 0x8d, 0x09, 0xec, 0x1c, 0xe2, 0x5d, 0xd1, 0x31, 0x47, 0x8a, 0x77, 0xa5, - 0x0c, 0x57, 0x12, 0xe3, 0x6c, 0xfa, 0xec, 0xe4, 0xb9, 0xf8, 0xdb, 0xed, 0x9b, 0xff, 0xfe, 0x66, - 0x5f, 0x77, 0xf7, 0x47, 0xbb, 0xe4, 0x97, 0x1b, 0x0c, 0xf5, 0x75, 0xdd, 0xe2, 0x7f, 0xdd, 0xf0, - 0x0c, 0xfd, 0x06, 0xa5, 0xbe, 0x41, 0xf6, 0x18, 0xee, 0xee, 0x96, 0xe8, 0xea, 0xe6, 0x3f, 0x02, - 0x00, 0x00, 0xff, 0xff, 0xa0, 0x19, 0x51, 0xc3, 0xf7, 0x3f, 0x00, 0x00, + // 3659 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1b, 0xdb, 0x6e, 0x1b, 0xc7, + 0xd5, 0xcb, 0x9b, 0xc8, 0xc3, 0x8b, 0xa8, 0xb1, 0x23, 0xd3, 0xb4, 0x2d, 0xcb, 0xeb, 0xd8, 0x51, + 0x1c, 0xc7, 0x4e, 0xe4, 0x06, 0x09, 0x9a, 0x1b, 0x6c, 0xcb, 0x56, 0x88, 0x4a, 0xae, 0xbc, 0x52, + 0xe2, 0xa2, 0x29, 0x4a, 0xac, 0xb8, 0x23, 0x6a, 0x23, 0xee, 0x2e, 0xbd, 0xbb, 0xb4, 0xac, 0xbc, + 0xc4, 0x68, 0x80, 0x02, 0x29, 0x8a, 0xb6, 0x40, 0x5f, 0x5a, 0xa0, 0x05, 0x8a, 0x3e, 0xf5, 0x82, + 0x02, 0x05, 0x82, 0x3e, 0xa4, 0x45, 0xdf, 0x83, 0xf6, 0xa1, 0x1f, 0xd0, 0x0f, 0x68, 0x5f, 0xda, + 0x6f, 0x28, 0xe6, 0xb2, 0xb3, 0x77, 0x72, 0x45, 0xd9, 0x71, 0xdf, 0x38, 0x67, 0xcf, 0x9c, 0x39, + 0x73, 0xe6, 0xdc, 0x67, 0x08, 0x4d, 0x4d, 0x75, 0xd5, 0x6e, 0xcf, 0xb2, 0x6c, 0xed, 0xea, 0xd0, + 0xb6, 0x5c, 0x0b, 0xcd, 0x19, 0xfa, 0xe0, 0xe1, 0xc8, 0x61, 0xa3, 0xab, 0xe4, 0x73, 0xbb, 0xd6, + 0xb3, 0x0c, 0xc3, 0x32, 0x19, 0xa8, 0xdd, 0xd0, 0x4d, 0x17, 0xdb, 0xa6, 0x3a, 0xe0, 0xe3, 0x5a, + 0x70, 0x42, 0xbb, 0xe6, 0xf4, 0x76, 0xb1, 0xa1, 0xb2, 0x91, 0x3c, 0x03, 0xc5, 0xdb, 0xc6, 0xd0, + 0x3d, 0x90, 0x7f, 0x26, 0x41, 0xed, 0xce, 0x60, 0xe4, 0xec, 0x2a, 0xf8, 0xc1, 0x08, 0x3b, 0x2e, + 0x7a, 0x05, 0x0a, 0xdb, 0xaa, 0x83, 0x5b, 0xd2, 0xa2, 0xb4, 0x54, 0x5d, 0x3e, 0x73, 0x35, 0xb4, + 0x2a, 0x5f, 0x6f, 0xdd, 0xe9, 0xdf, 0x54, 0x1d, 0xac, 0x50, 0x4c, 0x84, 0xa0, 0xa0, 0x6d, 0x77, + 0x56, 0x5a, 0xb9, 0x45, 0x69, 0x29, 0xaf, 0xd0, 0xdf, 0x68, 0x01, 0xc0, 0xc1, 0x7d, 0x03, 0x9b, + 0x6e, 0x67, 0xc5, 0x69, 0xe5, 0x17, 0xf3, 0x4b, 0x79, 0x25, 0x00, 0x41, 0x32, 0xd4, 0x7a, 0xd6, + 0x60, 0x80, 0x7b, 0xae, 0x6e, 0x99, 0x9d, 0x95, 0x56, 0x81, 0xce, 0x0d, 0xc1, 0xe4, 0x5f, 0x48, + 0x50, 0xe7, 0xac, 0x39, 0x43, 0xcb, 0x74, 0x30, 0xba, 0x0e, 0x25, 0xc7, 0x55, 0xdd, 0x91, 0xc3, + 0xb9, 0x3b, 0x9d, 0xc8, 0xdd, 0x26, 0x45, 0x51, 0x38, 0x6a, 0x22, 0x7b, 0xd1, 0xe5, 0xf3, 0xf1, + 0xe5, 0x23, 0x5b, 0x28, 0x44, 0xb7, 0x20, 0xff, 0x41, 0x82, 0xe6, 0xa6, 0x37, 0xf4, 0xa4, 0x77, + 0x02, 0x8a, 0x3d, 0x6b, 0x64, 0xba, 0x94, 0xc1, 0xba, 0xc2, 0x06, 0xe8, 0x3c, 0xd4, 0x7a, 0xbb, + 0xaa, 0x69, 0xe2, 0x41, 0xd7, 0x54, 0x0d, 0x4c, 0x59, 0xa9, 0x28, 0x55, 0x0e, 0xbb, 0xab, 0x1a, + 0x38, 0x13, 0x47, 0x8b, 0x50, 0x1d, 0xaa, 0xb6, 0xab, 0x87, 0x64, 0x16, 0x04, 0xa1, 0x36, 0x94, + 0x75, 0xa7, 0x63, 0x0c, 0x2d, 0xdb, 0x6d, 0x15, 0x17, 0xa5, 0xa5, 0xb2, 0x22, 0xc6, 0xf2, 0xaf, + 0x24, 0x98, 0xbf, 0xe1, 0x38, 0x7a, 0xdf, 0x8c, 0x71, 0x3d, 0x0f, 0x25, 0xd3, 0xd2, 0x70, 0x67, + 0x85, 0xb2, 0x9d, 0x57, 0xf8, 0x08, 0x9d, 0x86, 0xca, 0x10, 0x63, 0xbb, 0x6b, 0x5b, 0x03, 0x8f, + 0xe9, 0x32, 0x01, 0x28, 0xd6, 0x00, 0xa3, 0x7b, 0x30, 0xe7, 0x44, 0x08, 0xb1, 0x93, 0xae, 0x2e, + 0x5f, 0xb8, 0x1a, 0xd3, 0xd5, 0xab, 0xd1, 0x45, 0x95, 0xf8, 0x6c, 0xf9, 0x71, 0x0e, 0x8e, 0x0b, + 0x3c, 0xc6, 0x2b, 0xf9, 0x4d, 0xa4, 0xea, 0xe0, 0xbe, 0x60, 0x8f, 0x0d, 0xb2, 0x48, 0x55, 0x1c, + 0x47, 0x3e, 0x78, 0x1c, 0x19, 0x94, 0x2f, 0x2a, 0xeb, 0x62, 0x5c, 0xd6, 0xe7, 0xa0, 0x8a, 0x1f, + 0x0d, 0x75, 0x1b, 0x77, 0x5d, 0xdd, 0xc0, 0xad, 0xd2, 0xa2, 0xb4, 0x54, 0x50, 0x80, 0x81, 0xb6, + 0x74, 0x23, 0xa8, 0xad, 0x33, 0x99, 0xb5, 0x55, 0xfe, 0xb5, 0x04, 0x27, 0x63, 0xa7, 0xc4, 0xd5, + 0x5f, 0x81, 0x26, 0xdd, 0xb9, 0x2f, 0x19, 0x62, 0x08, 0x44, 0xe0, 0x97, 0xc6, 0x09, 0xdc, 0x47, + 0x57, 0x62, 0xf3, 0x03, 0x4c, 0xe6, 0xb2, 0x33, 0xb9, 0x07, 0x27, 0x57, 0xb1, 0xcb, 0x17, 0x20, + 0xdf, 0xb0, 0x33, 0xbd, 0xfb, 0x08, 0xdb, 0x59, 0x2e, 0x66, 0x67, 0x7f, 0xcc, 0x09, 0x3b, 0xa3, + 0x4b, 0x75, 0xcc, 0x1d, 0x0b, 0x9d, 0x81, 0x8a, 0x40, 0xe1, 0x5a, 0xe1, 0x03, 0xd0, 0xeb, 0x50, + 0x24, 0x9c, 0x32, 0x95, 0x68, 0x2c, 0x9f, 0x4f, 0xde, 0x53, 0x80, 0xa6, 0xc2, 0xf0, 0x51, 0x07, + 0x1a, 0x8e, 0xab, 0xda, 0x6e, 0x77, 0x68, 0x39, 0xf4, 0x9c, 0xa9, 0xe2, 0x54, 0x97, 0xe5, 0x30, + 0x05, 0xe1, 0x68, 0xd7, 0x9d, 0xfe, 0x06, 0xc7, 0x54, 0xea, 0x74, 0xa6, 0x37, 0x44, 0xb7, 0xa1, + 0x86, 0x4d, 0xcd, 0x27, 0x54, 0xc8, 0x4c, 0xa8, 0x8a, 0x4d, 0x4d, 0x90, 0xf1, 0xcf, 0xa7, 0x98, + 0xfd, 0x7c, 0x7e, 0x28, 0x41, 0x2b, 0x7e, 0x40, 0x47, 0x71, 0xa2, 0x6f, 0xb2, 0x49, 0x98, 0x1d, + 0xd0, 0x58, 0x0b, 0x17, 0x87, 0xa4, 0xf0, 0x29, 0xb2, 0x0e, 0xcf, 0xf9, 0xdc, 0xd0, 0x2f, 0x4f, + 0x4d, 0x59, 0x3e, 0x95, 0x60, 0x3e, 0xba, 0xd6, 0x51, 0xf6, 0xfd, 0x35, 0x28, 0xea, 0xe6, 0x8e, + 0xe5, 0x6d, 0x7b, 0x61, 0x8c, 0x9d, 0x91, 0xb5, 0x18, 0xb2, 0x6c, 0xc0, 0xe9, 0x55, 0xec, 0x76, + 0x4c, 0x07, 0xdb, 0xee, 0x4d, 0xdd, 0x1c, 0x58, 0xfd, 0x0d, 0xd5, 0xdd, 0x3d, 0x82, 0x8d, 0x84, + 0xd4, 0x3d, 0x17, 0x51, 0x77, 0xf9, 0x37, 0x12, 0x9c, 0x49, 0x5e, 0x8f, 0x6f, 0xbd, 0x0d, 0xe5, + 0x1d, 0x1d, 0x0f, 0x34, 0x22, 0x33, 0x89, 0xca, 0x4c, 0x8c, 0x89, 0xad, 0x0c, 0x09, 0x32, 0xdf, + 0xe1, 0xf9, 0x14, 0x05, 0xdd, 0x74, 0x6d, 0xdd, 0xec, 0xaf, 0xe9, 0x8e, 0xab, 0x30, 0xfc, 0x80, + 0x3c, 0xf3, 0xd9, 0x35, 0xf3, 0x07, 0x12, 0x2c, 0xac, 0x62, 0xf7, 0x96, 0x70, 0xb5, 0xe4, 0xbb, + 0xee, 0xb8, 0x7a, 0xcf, 0x79, 0xb2, 0x09, 0x48, 0x86, 0x78, 0x2a, 0xff, 0x58, 0x82, 0x73, 0xa9, + 0xcc, 0x70, 0xd1, 0x71, 0x57, 0xe2, 0x39, 0xda, 0x64, 0x57, 0xf2, 0x0d, 0x7c, 0xf0, 0x81, 0x3a, + 0x18, 0xe1, 0x0d, 0x55, 0xb7, 0x99, 0x2b, 0x99, 0xd2, 0xb1, 0xfe, 0x5e, 0x82, 0xb3, 0xab, 0xd8, + 0xdd, 0xf0, 0xc2, 0xcc, 0x33, 0x94, 0xce, 0xe4, 0x6c, 0x43, 0xfe, 0x11, 0x3b, 0xcc, 0x44, 0x6e, + 0x9f, 0x89, 0xf8, 0x16, 0xa8, 0x1d, 0x04, 0x0c, 0xf2, 0x16, 0xcb, 0x05, 0xb8, 0xf0, 0xe4, 0x5f, + 0x4a, 0x70, 0xea, 0x46, 0xef, 0xc1, 0x48, 0xb7, 0x31, 0x47, 0x5a, 0xb3, 0x7a, 0x7b, 0xd3, 0x8b, + 0xd6, 0xcf, 0x9b, 0x72, 0xa1, 0xbc, 0x69, 0x52, 0xf6, 0x3b, 0x0f, 0x25, 0x57, 0x75, 0xf6, 0x84, + 0x54, 0xf9, 0x88, 0xf2, 0xa7, 0xe0, 0x01, 0x56, 0x9d, 0xff, 0x4f, 0xfe, 0x1e, 0xe7, 0xa1, 0xf6, + 0x01, 0xcf, 0xaf, 0x68, 0x18, 0x8e, 0xea, 0x91, 0x94, 0xac, 0x47, 0x81, 0x94, 0x2c, 0x29, 0x4b, + 0x5b, 0x85, 0xba, 0x83, 0xf1, 0xde, 0x34, 0x41, 0xb7, 0x46, 0x26, 0x8a, 0x60, 0xb9, 0x06, 0x73, + 0x23, 0x73, 0x87, 0x94, 0x0c, 0x58, 0xe3, 0x02, 0x64, 0x99, 0xfb, 0x64, 0xcf, 0x1d, 0x9f, 0x88, + 0xde, 0x83, 0xd9, 0x28, 0xad, 0x62, 0x26, 0x5a, 0xd1, 0x69, 0xa8, 0x03, 0x4d, 0xcd, 0xb6, 0x86, + 0x43, 0xac, 0x75, 0x1d, 0x8f, 0x54, 0x29, 0x1b, 0x29, 0x3e, 0xcf, 0x23, 0x25, 0x7f, 0x26, 0xc1, + 0xfc, 0x7d, 0xd5, 0xed, 0xed, 0xae, 0x18, 0x5c, 0xb9, 0x8f, 0xe0, 0x1a, 0xde, 0x86, 0xca, 0x43, + 0x7e, 0x10, 0x9e, 0xff, 0x3f, 0x97, 0xc0, 0x50, 0xf0, 0xc8, 0x15, 0x7f, 0x86, 0xfc, 0xa5, 0x04, + 0x27, 0x68, 0x81, 0xe6, 0x71, 0xf7, 0xd5, 0x3b, 0xa9, 0x09, 0x45, 0x1a, 0xba, 0x04, 0x0d, 0x43, + 0xb5, 0xf7, 0x36, 0x7d, 0x9c, 0x22, 0xc5, 0x89, 0x40, 0xe5, 0x47, 0x00, 0x7c, 0xb4, 0xee, 0xf4, + 0xa7, 0xe0, 0xff, 0x0d, 0x98, 0xe1, 0xab, 0x72, 0x7f, 0x35, 0xe9, 0x60, 0x3d, 0x74, 0xf9, 0x6f, + 0x12, 0x34, 0xfc, 0x08, 0x44, 0xad, 0xaa, 0x01, 0x39, 0x61, 0x4b, 0xb9, 0xce, 0x0a, 0x7a, 0x1b, + 0x4a, 0xac, 0x78, 0xe7, 0xb4, 0x2f, 0x86, 0x69, 0xf3, 0xc2, 0x3e, 0x10, 0xc6, 0x28, 0x40, 0xe1, + 0x93, 0x88, 0x8c, 0x84, 0xd7, 0x16, 0xd6, 0xee, 0x43, 0x50, 0x07, 0x66, 0xc3, 0x49, 0xaf, 0x67, + 0x33, 0x8b, 0x69, 0xde, 0x7a, 0x45, 0x75, 0x55, 0xea, 0xac, 0x1b, 0xa1, 0x9c, 0xd7, 0x91, 0xff, + 0x5b, 0x84, 0x6a, 0x60, 0x97, 0xb1, 0x9d, 0x44, 0x8f, 0x34, 0x37, 0x39, 0xee, 0xe4, 0xe3, 0x95, + 0xd7, 0x45, 0x68, 0xe8, 0x34, 0xd7, 0xe9, 0x72, 0x55, 0xa4, 0x6e, 0xaa, 0xa2, 0xd4, 0x19, 0x94, + 0xdb, 0x05, 0x5a, 0x80, 0xaa, 0x39, 0x32, 0xba, 0xd6, 0x4e, 0xd7, 0xb6, 0xf6, 0x1d, 0x5e, 0xc2, + 0x55, 0xcc, 0x91, 0xf1, 0xcd, 0x1d, 0xc5, 0xda, 0x77, 0xfc, 0x2a, 0xa1, 0x74, 0xc8, 0x2a, 0x61, + 0x01, 0xaa, 0x86, 0xfa, 0x88, 0x50, 0xed, 0x9a, 0x23, 0x83, 0x56, 0x77, 0x79, 0xa5, 0x62, 0xa8, + 0x8f, 0x14, 0x6b, 0xff, 0xee, 0xc8, 0x40, 0x4b, 0xd0, 0x1c, 0xa8, 0x8e, 0xdb, 0x0d, 0x96, 0x87, + 0x65, 0x5a, 0x1e, 0x36, 0x08, 0xfc, 0xb6, 0x5f, 0x22, 0xc6, 0xeb, 0x8d, 0xca, 0x11, 0xea, 0x0d, + 0xcd, 0x18, 0xf8, 0x84, 0x20, 0x7b, 0xbd, 0xa1, 0x19, 0x03, 0x41, 0xe6, 0x0d, 0x98, 0xd9, 0xa6, + 0x19, 0xa4, 0xd3, 0xaa, 0xa6, 0x7a, 0xa8, 0x3b, 0x24, 0x79, 0x64, 0x89, 0xa6, 0xe2, 0xa1, 0xa3, + 0xb7, 0xa0, 0x42, 0x43, 0x37, 0x9d, 0x5b, 0xcb, 0x34, 0xd7, 0x9f, 0x40, 0x66, 0x6b, 0x78, 0xe0, + 0xaa, 0x74, 0x76, 0x3d, 0xdb, 0x6c, 0x31, 0x01, 0xbd, 0x02, 0xc7, 0x7b, 0x36, 0x56, 0x5d, 0xac, + 0xdd, 0x3c, 0xb8, 0x65, 0x19, 0x43, 0x95, 0x2a, 0x53, 0xab, 0x41, 0x5b, 0x20, 0x49, 0x9f, 0x88, + 0x63, 0xe8, 0x89, 0xd1, 0x1d, 0xdb, 0x32, 0x5a, 0xb3, 0xcc, 0x31, 0x84, 0xa1, 0xe8, 0x2c, 0x80, + 0xe7, 0xba, 0x55, 0xb7, 0xd5, 0xa4, 0xa7, 0x58, 0xe1, 0x90, 0x1b, 0xae, 0xfc, 0x09, 0x9c, 0xf0, + 0x35, 0x24, 0x70, 0x1a, 0xf1, 0x83, 0x95, 0xa6, 0x3d, 0xd8, 0xf1, 0xb9, 0xff, 0x3f, 0x0a, 0x30, + 0xbf, 0xa9, 0x3e, 0xc4, 0x4f, 0xbf, 0xcc, 0xc8, 0xe4, 0x8f, 0xd7, 0x60, 0x8e, 0x56, 0x16, 0xcb, + 0x01, 0x7e, 0xc6, 0x44, 0xe0, 0xe0, 0x71, 0xc6, 0x27, 0xa2, 0x77, 0x49, 0xea, 0x80, 0x7b, 0x7b, + 0x1b, 0x96, 0xee, 0x47, 0xdf, 0xb3, 0x09, 0x74, 0x6e, 0x09, 0x2c, 0x25, 0x38, 0x03, 0x6d, 0xc4, + 0x5d, 0x1b, 0x8b, 0xbb, 0x2f, 0x8c, 0xad, 0x5f, 0x7d, 0xe9, 0x47, 0x3d, 0x1c, 0x6a, 0xc1, 0x0c, + 0x8f, 0xee, 0xd4, 0xee, 0xcb, 0x8a, 0x37, 0x44, 0x1b, 0x70, 0x9c, 0xed, 0x60, 0x93, 0x2b, 0x35, + 0xdb, 0x7c, 0x39, 0xd3, 0xe6, 0x93, 0xa6, 0x86, 0x6d, 0xa2, 0x72, 0x58, 0x9b, 0x68, 0xc1, 0x0c, + 0xd7, 0x53, 0xea, 0x0b, 0xca, 0x8a, 0x37, 0x24, 0xc7, 0xac, 0xd3, 0x9e, 0xa0, 0x6e, 0xf6, 0x5b, + 0x55, 0xfa, 0xcd, 0x07, 0x90, 0x12, 0x0d, 0x7c, 0x79, 0x4e, 0xe8, 0xb4, 0xbc, 0x03, 0x65, 0xa1, + 0xe1, 0xb9, 0xcc, 0x1a, 0x2e, 0xe6, 0x44, 0x7d, 0x74, 0x3e, 0xe2, 0xa3, 0xe5, 0xbf, 0x4b, 0x50, + 0x5b, 0x21, 0x5b, 0x5a, 0xb3, 0xfa, 0x34, 0xa2, 0x5c, 0x84, 0x86, 0x8d, 0x7b, 0x96, 0xad, 0x75, + 0xb1, 0xe9, 0xda, 0x3a, 0x66, 0xd5, 0x7c, 0x41, 0xa9, 0x33, 0xe8, 0x6d, 0x06, 0x24, 0x68, 0xc4, + 0xed, 0x3a, 0xae, 0x6a, 0x0c, 0xbb, 0x3b, 0xc4, 0xbc, 0x73, 0x0c, 0x4d, 0x40, 0xa9, 0x75, 0x9f, + 0x87, 0x9a, 0x8f, 0xe6, 0x5a, 0x74, 0xfd, 0x82, 0x52, 0x15, 0xb0, 0x2d, 0x0b, 0x3d, 0x0f, 0x0d, + 0x2a, 0xd3, 0xee, 0xc0, 0xea, 0x77, 0x49, 0xe5, 0xcb, 0x83, 0x4d, 0x4d, 0xe3, 0x6c, 0x91, 0xb3, + 0x0a, 0x63, 0x39, 0xfa, 0xc7, 0x98, 0x87, 0x1b, 0x81, 0xb5, 0xa9, 0x7f, 0x8c, 0x49, 0xac, 0xaf, + 0x93, 0xd8, 0x79, 0xd7, 0xd2, 0xf0, 0xd6, 0x94, 0x99, 0x46, 0x86, 0xae, 0xe7, 0x19, 0xa8, 0x88, + 0x1d, 0xf0, 0x2d, 0xf9, 0x00, 0x74, 0x07, 0x1a, 0x5e, 0x12, 0xda, 0x65, 0xb5, 0x59, 0x21, 0x35, + 0xf3, 0x0b, 0x44, 0x3f, 0x47, 0xa9, 0x7b, 0xd3, 0xe8, 0x50, 0xbe, 0x03, 0xb5, 0xe0, 0x67, 0xb2, + 0xea, 0x66, 0x54, 0x51, 0x04, 0x80, 0x68, 0xe3, 0xdd, 0x91, 0x41, 0xce, 0x94, 0x3b, 0x16, 0x6f, + 0x28, 0x7f, 0x2a, 0x41, 0x9d, 0x87, 0xec, 0x4d, 0xd1, 0xb1, 0xa7, 0x5b, 0x93, 0xe8, 0xd6, 0xe8, + 0x6f, 0xf4, 0xf5, 0x70, 0x4b, 0xef, 0xf9, 0x44, 0x27, 0x40, 0x89, 0xd0, 0xec, 0x38, 0x14, 0xaf, + 0xb3, 0xf4, 0x02, 0x1e, 0x13, 0x45, 0xe3, 0x47, 0x43, 0x15, 0xad, 0x05, 0x33, 0xaa, 0xa6, 0xd9, + 0xd8, 0x71, 0x38, 0x1f, 0xde, 0x90, 0x7c, 0x79, 0x88, 0x6d, 0xc7, 0x53, 0xf9, 0xbc, 0xe2, 0x0d, + 0xd1, 0x5b, 0x50, 0x16, 0xe9, 0x74, 0x3e, 0x29, 0x85, 0x0a, 0xf2, 0xc9, 0x6b, 0x57, 0x31, 0x43, + 0xfe, 0x53, 0x0e, 0x1a, 0x5c, 0x60, 0x37, 0x79, 0x4c, 0x1d, 0x6f, 0x7c, 0x37, 0xa1, 0xb6, 0xe3, + 0xdb, 0xfe, 0xb8, 0x1e, 0x55, 0xd0, 0x45, 0x84, 0xe6, 0x4c, 0x32, 0xc0, 0x70, 0x54, 0x2f, 0x1c, + 0x29, 0xaa, 0x17, 0x0f, 0xeb, 0xc1, 0xe2, 0x79, 0x5e, 0x29, 0x21, 0xcf, 0x93, 0xbf, 0x03, 0xd5, + 0x00, 0x01, 0xea, 0xa1, 0x59, 0x73, 0x8b, 0x4b, 0xcc, 0x1b, 0xa2, 0xeb, 0x7e, 0x6e, 0xc3, 0x44, + 0x75, 0x2a, 0x81, 0x97, 0x48, 0x5a, 0x23, 0xff, 0x56, 0x82, 0x12, 0xa7, 0x7c, 0x0e, 0xaa, 0xdc, + 0xe9, 0xd0, 0xbc, 0x8f, 0x51, 0x07, 0x0e, 0x22, 0x89, 0xdf, 0x93, 0xf3, 0x3a, 0xa7, 0xa0, 0x1c, + 0xf1, 0x37, 0x33, 0x3c, 0x2c, 0x78, 0x9f, 0x02, 0x4e, 0x86, 0x7c, 0xa2, 0xfe, 0xe5, 0x4b, 0x89, + 0x36, 0xe6, 0x15, 0xdc, 0xb3, 0x1e, 0x62, 0xfb, 0xe0, 0xe8, 0xed, 0xcf, 0x37, 0x03, 0x0a, 0x9d, + 0xb1, 0x3e, 0x14, 0x13, 0xd0, 0x9b, 0xbe, 0xb8, 0xf3, 0x49, 0xdd, 0x9f, 0xa0, 0x87, 0xe1, 0xea, + 0xe8, 0x8b, 0xfd, 0x27, 0xac, 0x91, 0x1b, 0xde, 0xca, 0xb4, 0x79, 0xcd, 0x13, 0x29, 0x3b, 0xe4, + 0x9f, 0x4a, 0x70, 0x6a, 0x15, 0xbb, 0x77, 0xc2, 0xc5, 0xfd, 0xb3, 0xe6, 0xca, 0x80, 0x76, 0x12, + 0x53, 0x47, 0x39, 0xf5, 0x36, 0x94, 0x45, 0x9b, 0x82, 0xb5, 0xd8, 0xc5, 0x58, 0xfe, 0xbe, 0x04, + 0x2d, 0xbe, 0x0a, 0x5d, 0x93, 0xa4, 0xd4, 0x03, 0xec, 0x62, 0xed, 0xab, 0xae, 0x9b, 0xff, 0x2a, + 0x41, 0x33, 0xe8, 0xf1, 0xa9, 0xd3, 0x7e, 0x0d, 0x8a, 0xb4, 0x3d, 0xc1, 0x39, 0x98, 0xa8, 0xac, + 0x0c, 0x9b, 0xb8, 0x0c, 0x9a, 0xe6, 0x6d, 0x89, 0xe0, 0xc4, 0x87, 0x7e, 0xd8, 0xc9, 0x1f, 0x3e, + 0xec, 0xf0, 0x30, 0x6c, 0x8d, 0x08, 0x5d, 0xd6, 0x48, 0xf3, 0x01, 0xf2, 0xe7, 0x39, 0x68, 0xf9, + 0xf5, 0xc8, 0x57, 0xee, 0xf7, 0x53, 0xb2, 0xd5, 0xfc, 0x13, 0xca, 0x56, 0x0b, 0x47, 0xf7, 0xf5, + 0xc5, 0x24, 0x5f, 0xff, 0x97, 0x1c, 0x34, 0x7c, 0xa9, 0x6d, 0x0c, 0x54, 0x13, 0xcd, 0x43, 0x69, + 0x38, 0x50, 0xfd, 0xee, 0x23, 0x1f, 0xa1, 0x4d, 0x91, 0xe7, 0x84, 0xe5, 0xf4, 0x52, 0xd2, 0x19, + 0xa6, 0x1c, 0x84, 0x12, 0x21, 0x41, 0xca, 0x41, 0x56, 0x50, 0xd0, 0xa2, 0x9e, 0xe7, 0x56, 0x4c, + 0x59, 0x48, 0x3d, 0x7f, 0x05, 0x10, 0x3f, 0xe1, 0xae, 0x6e, 0x76, 0x1d, 0xdc, 0xb3, 0x4c, 0x8d, + 0x9d, 0x7d, 0x51, 0x69, 0xf2, 0x2f, 0x1d, 0x73, 0x93, 0xc1, 0xd1, 0x6b, 0x50, 0x70, 0x0f, 0x86, + 0xcc, 0x8b, 0x37, 0x12, 0xbd, 0xa3, 0xcf, 0xd7, 0xd6, 0xc1, 0x10, 0x2b, 0x14, 0x1d, 0x2d, 0x00, + 0x10, 0x52, 0xae, 0xad, 0x3e, 0xe4, 0x21, 0xb1, 0xa0, 0x04, 0x20, 0x44, 0x9b, 0x3d, 0x19, 0xce, + 0xb0, 0xd0, 0xc1, 0x87, 0xf2, 0x17, 0x39, 0x68, 0xfa, 0x24, 0x15, 0xec, 0x8c, 0x06, 0x6e, 0xaa, + 0xfc, 0xc6, 0x17, 0x83, 0x93, 0xf2, 0x86, 0x77, 0xa1, 0xca, 0xcf, 0xf3, 0x10, 0xfa, 0x00, 0x6c, + 0xca, 0xda, 0x18, 0x05, 0x2d, 0x3e, 0x21, 0x05, 0x2d, 0x1d, 0x52, 0x41, 0xe5, 0x4d, 0x98, 0xf7, + 0xfc, 0x9e, 0x8f, 0xb0, 0x8e, 0x5d, 0x75, 0x4c, 0xc2, 0x71, 0x0e, 0xaa, 0x2c, 0x9e, 0xb1, 0x40, + 0xce, 0x52, 0x75, 0xd8, 0x16, 0x15, 0xae, 0xfc, 0x5d, 0x38, 0x41, 0xfd, 0x46, 0xb4, 0x95, 0x9b, + 0xa5, 0xaf, 0x2e, 0x8b, 0x42, 0x80, 0x24, 0xfd, 0x4c, 0xbb, 0x2b, 0x4a, 0x08, 0x26, 0xaf, 0xc1, + 0x73, 0x11, 0xfa, 0x47, 0x88, 0x0b, 0x24, 0x15, 0x9a, 0xdf, 0x0c, 0x5f, 0x2b, 0x4f, 0x1f, 0xfd, + 0xce, 0x8a, 0xce, 0x6d, 0x57, 0xd7, 0xa2, 0xfa, 0xa5, 0xa1, 0x77, 0xa0, 0x62, 0xe2, 0xfd, 0x6e, + 0xd0, 0xf9, 0x66, 0x68, 0xd0, 0x95, 0x4d, 0xbc, 0x4f, 0x7f, 0xc9, 0x77, 0xe1, 0x64, 0x8c, 0xd5, + 0xa3, 0xec, 0xfd, 0xcf, 0x12, 0x9c, 0x5a, 0xb1, 0xad, 0xe1, 0x07, 0xba, 0xed, 0x8e, 0xd4, 0x41, + 0xf8, 0x62, 0xe9, 0xe9, 0x94, 0x71, 0xef, 0x05, 0xc2, 0x30, 0xf3, 0xcb, 0x57, 0x12, 0xd4, 0x35, + 0xce, 0x14, 0xdf, 0x74, 0x20, 0x68, 0xff, 0x2b, 0x9f, 0xc4, 0x3c, 0xc7, 0x9b, 0x10, 0x6c, 0xb2, + 0x64, 0x29, 0x89, 0x5d, 0x9f, 0xfc, 0xb4, 0x5d, 0x9f, 0x14, 0xcb, 0x2f, 0x3c, 0x21, 0xcb, 0x3f, + 0x74, 0x19, 0xf2, 0x1e, 0x84, 0x3b, 0x72, 0xd4, 0xe5, 0x4e, 0xd5, 0xca, 0xbb, 0x09, 0xe0, 0x77, + 0xa7, 0xf8, 0xab, 0xa0, 0x2c, 0x64, 0x02, 0xb3, 0xc8, 0x69, 0x09, 0x2f, 0x4b, 0xbb, 0xca, 0xa1, + 0x7e, 0xc9, 0x3d, 0x68, 0x27, 0x69, 0xe9, 0x51, 0x34, 0xff, 0xf3, 0x1c, 0x00, 0x7b, 0x42, 0xb6, + 0xa5, 0x3a, 0x7b, 0xd3, 0x65, 0x94, 0x17, 0xa0, 0xee, 0x2b, 0x8c, 0x6f, 0xef, 0x41, 0x2d, 0xd2, + 0x88, 0x49, 0x88, 0xc4, 0x96, 0xe0, 0xc4, 0x92, 0x5d, 0x8d, 0xd2, 0x09, 0x58, 0x0d, 0x53, 0x8a, + 0x88, 0xd3, 0x43, 0xa7, 0xa1, 0x62, 0x5b, 0xfb, 0x5d, 0x62, 0x66, 0x9a, 0xf7, 0x0a, 0xce, 0xb6, + 0xf6, 0x89, 0xf1, 0x69, 0xe8, 0x24, 0xcc, 0xb8, 0xaa, 0xb3, 0x47, 0xe8, 0x97, 0x02, 0x77, 0x9b, + 0x1a, 0x3a, 0x01, 0xc5, 0x1d, 0x7d, 0x80, 0x9d, 0xd6, 0x0c, 0x25, 0xc9, 0x06, 0xe8, 0x75, 0xef, + 0xfd, 0x47, 0x39, 0xf3, 0xfd, 0x35, 0x7b, 0x02, 0xf2, 0xa5, 0x04, 0xb3, 0xbe, 0xd4, 0xa8, 0x03, + 0x22, 0x3e, 0x8d, 0xfa, 0xb3, 0x5b, 0x96, 0xc6, 0x5c, 0x45, 0x23, 0xe5, 0x8a, 0x85, 0x4d, 0x64, + 0x5e, 0xcb, 0x9f, 0x32, 0x2e, 0x2f, 0x27, 0xfb, 0x22, 0x9b, 0xd6, 0x35, 0xef, 0x86, 0xa7, 0x64, + 0x5b, 0xfb, 0x1d, 0x4d, 0x48, 0x83, 0x3d, 0x83, 0x63, 0x59, 0x28, 0x91, 0xc6, 0x2d, 0xfa, 0x12, + 0xee, 0x02, 0xd4, 0xb1, 0x6d, 0x5b, 0x76, 0xd7, 0xc0, 0x8e, 0xa3, 0xf6, 0x31, 0x4f, 0xba, 0x6a, + 0x14, 0xb8, 0xce, 0x60, 0xf2, 0x17, 0x79, 0x68, 0xf8, 0x5b, 0xf1, 0xee, 0x75, 0x74, 0xcd, 0xbb, + 0xd7, 0xd1, 0x35, 0xe2, 0xcc, 0x6d, 0xe6, 0x0a, 0x03, 0xce, 0x9c, 0x43, 0x3a, 0x1a, 0x89, 0x83, + 0xc4, 0xc0, 0x4c, 0x4b, 0xc3, 0xfe, 0xc1, 0x82, 0x07, 0xe2, 0xe7, 0x1a, 0xd2, 0x8f, 0x42, 0x06, + 0xfd, 0x28, 0x66, 0xd0, 0x8f, 0x52, 0x82, 0x7e, 0xcc, 0x43, 0x69, 0x7b, 0xd4, 0xdb, 0xc3, 0x2e, + 0x4f, 0x8f, 0xf8, 0x28, 0xac, 0x37, 0xe5, 0x88, 0xde, 0x08, 0xf5, 0xa8, 0x04, 0xd5, 0xe3, 0x34, + 0x54, 0xd8, 0xe5, 0x42, 0xd7, 0x75, 0x68, 0x97, 0x35, 0xaf, 0x94, 0x19, 0x60, 0xcb, 0x41, 0x6f, + 0x78, 0xb5, 0x43, 0x35, 0xc9, 0xd0, 0xa9, 0xc7, 0x89, 0x68, 0x88, 0x57, 0x39, 0x5c, 0x84, 0x06, + 0x7d, 0xfe, 0xfb, 0x60, 0x84, 0xed, 0x03, 0x75, 0x7b, 0x80, 0x5b, 0x35, 0xca, 0x4e, 0x9d, 0x40, + 0xef, 0x79, 0x40, 0x22, 0x10, 0x8a, 0xa6, 0x9b, 0x1a, 0x7e, 0x84, 0xb5, 0x56, 0x9d, 0x22, 0x51, + 0x51, 0x77, 0x18, 0x48, 0xfe, 0x08, 0x90, 0xbf, 0xc6, 0xd1, 0xaa, 0xc2, 0xc8, 0x21, 0xe6, 0xa2, + 0x87, 0x28, 0xff, 0x4e, 0x82, 0xb9, 0xe0, 0x62, 0xd3, 0x86, 0xc6, 0x77, 0xa0, 0xca, 0xba, 0xd1, + 0x5d, 0x62, 0x9a, 0xbc, 0x2e, 0x3c, 0x3b, 0x56, 0x7a, 0x0a, 0xe8, 0xbe, 0x87, 0xba, 0x00, 0xf5, + 0x7d, 0xcb, 0xde, 0xd3, 0xcd, 0x7e, 0x97, 0x70, 0xe6, 0x19, 0x44, 0x8d, 0x03, 0xef, 0x12, 0x98, + 0xfc, 0x99, 0x04, 0x0b, 0xef, 0x0f, 0x35, 0xd5, 0xc5, 0x81, 0x1c, 0xe1, 0xa8, 0x4f, 0x6d, 0x5e, + 0xf3, 0x5e, 0xbb, 0xe4, 0xb2, 0x75, 0x54, 0x19, 0xb6, 0xbc, 0x0e, 0xa7, 0x14, 0xec, 0x60, 0x53, + 0x0b, 0x7d, 0x9c, 0x96, 0x0b, 0x79, 0x08, 0xed, 0x24, 0x72, 0x47, 0x39, 0x7b, 0x96, 0xac, 0x75, + 0x6d, 0x42, 0xd6, 0xe5, 0xbe, 0x87, 0xe4, 0x08, 0x74, 0x1d, 0x57, 0xfe, 0xb7, 0x04, 0x73, 0x37, + 0x34, 0x6f, 0xbd, 0xa7, 0x96, 0x13, 0x46, 0x73, 0xa6, 0x7c, 0x3c, 0x67, 0x7a, 0x52, 0x8e, 0x84, + 0xbb, 0x53, 0x73, 0x64, 0x78, 0x61, 0xc2, 0xa6, 0x77, 0xbb, 0xf2, 0x8e, 0xb8, 0xf0, 0x53, 0xf0, + 0x0e, 0xb6, 0xb1, 0xd9, 0xc3, 0x6b, 0x56, 0x6f, 0x2f, 0xf0, 0x64, 0x46, 0x0a, 0x3e, 0x99, 0x99, + 0xf6, 0x09, 0xce, 0xe5, 0x9f, 0x4b, 0x30, 0x17, 0xeb, 0x2c, 0xa0, 0x06, 0xc0, 0xfb, 0x66, 0x8f, + 0xb7, 0x5c, 0x9a, 0xc7, 0x50, 0x0d, 0xca, 0x5e, 0x03, 0xa6, 0x29, 0xa1, 0x2a, 0xcc, 0x6c, 0x59, + 0x14, 0xbb, 0x99, 0x43, 0x4d, 0xa8, 0xb1, 0x89, 0xa3, 0x5e, 0x0f, 0x3b, 0x4e, 0x33, 0x2f, 0x20, + 0x77, 0x54, 0x7d, 0x30, 0xb2, 0x71, 0xb3, 0x80, 0xea, 0x50, 0xd9, 0xb2, 0xf8, 0x83, 0xa3, 0x66, + 0x11, 0x21, 0x68, 0x78, 0xaf, 0x8f, 0xf8, 0xa4, 0x52, 0x00, 0xe6, 0x4d, 0x9b, 0xb9, 0xbc, 0x13, + 0xac, 0xc1, 0x49, 0x61, 0x8a, 0x4e, 0xc2, 0xf1, 0xf7, 0x4d, 0x0d, 0xef, 0xe8, 0x26, 0xd6, 0xfc, + 0x4f, 0xcd, 0x63, 0xe8, 0x38, 0xcc, 0x76, 0x4c, 0x13, 0xdb, 0x01, 0xa0, 0x44, 0x80, 0xeb, 0xd8, + 0xee, 0xe3, 0x00, 0x30, 0x87, 0xe6, 0xa0, 0xbe, 0xae, 0x3f, 0x0a, 0x80, 0xf2, 0xcb, 0xff, 0x3c, + 0x09, 0x95, 0x15, 0xd5, 0x55, 0x6f, 0x59, 0x96, 0xad, 0xa1, 0x21, 0x20, 0xfa, 0x58, 0xcf, 0x18, + 0x5a, 0xa6, 0x78, 0xd5, 0x8a, 0x5e, 0x49, 0x49, 0x9f, 0xe2, 0xa8, 0x5c, 0x2d, 0xdb, 0x97, 0x52, + 0x66, 0x44, 0xd0, 0xe5, 0x63, 0xc8, 0xa0, 0x2b, 0x92, 0xc2, 0x7e, 0x4b, 0xef, 0xed, 0x79, 0xcf, + 0x0a, 0xc6, 0xac, 0x18, 0x41, 0xf5, 0x56, 0x8c, 0x3c, 0x96, 0xe5, 0x03, 0xf6, 0xa2, 0xd2, 0xb3, + 0x4b, 0xf9, 0x18, 0x7a, 0x00, 0x27, 0x56, 0x71, 0xc0, 0x0f, 0x79, 0x0b, 0x2e, 0xa7, 0x2f, 0x18, + 0x43, 0x3e, 0xe4, 0x92, 0x6b, 0x50, 0xa4, 0x5d, 0x3c, 0x94, 0xe4, 0xaa, 0x82, 0x7f, 0x0b, 0x69, + 0x2f, 0xa6, 0x23, 0x08, 0x6a, 0x1f, 0xc1, 0x6c, 0xe4, 0xe9, 0x3a, 0x7a, 0x31, 0x61, 0x5a, 0xf2, + 0x9f, 0x10, 0xda, 0x97, 0xb3, 0xa0, 0x8a, 0xb5, 0xfa, 0xd0, 0x08, 0x3f, 0xf5, 0x43, 0x4b, 0x09, + 0xf3, 0x13, 0x9f, 0x1d, 0xb7, 0x5f, 0xcc, 0x80, 0x29, 0x16, 0x32, 0xa0, 0x19, 0x7d, 0x4a, 0x8d, + 0x2e, 0x8f, 0x25, 0x10, 0x56, 0xb7, 0x97, 0x32, 0xe1, 0x8a, 0xe5, 0x0e, 0xa8, 0x12, 0xc4, 0x9e, + 0xf2, 0xa2, 0xab, 0xc9, 0x64, 0xd2, 0xde, 0x18, 0xb7, 0xaf, 0x65, 0xc6, 0x17, 0x4b, 0x7f, 0x8f, + 0xdd, 0x1e, 0x24, 0x3d, 0x87, 0x45, 0xaf, 0x26, 0x93, 0x1b, 0xf3, 0x8e, 0xb7, 0xbd, 0x7c, 0x98, + 0x29, 0x82, 0x89, 0x4f, 0x68, 0xdb, 0x3f, 0xe1, 0x49, 0x69, 0xd4, 0xee, 0x3c, 0x7a, 0xe9, 0x6f, + 0x65, 0xdb, 0xaf, 0x1e, 0x62, 0x86, 0x60, 0xc0, 0x8a, 0x3e, 0x56, 0xf7, 0xcc, 0xf0, 0xda, 0x44, + 0xad, 0x99, 0xce, 0x06, 0x3f, 0x84, 0xd9, 0xc8, 0x03, 0x8e, 0x44, 0xab, 0x49, 0x7e, 0xe4, 0xd1, + 0x1e, 0x17, 0xbe, 0x99, 0x49, 0x46, 0x6e, 0x51, 0x50, 0x8a, 0xf6, 0x27, 0xdc, 0xb4, 0xb4, 0x2f, + 0x67, 0x41, 0x15, 0x1b, 0x71, 0xa8, 0xbb, 0x8c, 0xdc, 0x44, 0xa0, 0x2b, 0xc9, 0x34, 0x92, 0x6f, + 0x51, 0xda, 0x2f, 0x67, 0xc4, 0x16, 0x8b, 0x76, 0x01, 0x56, 0xb1, 0xbb, 0x8e, 0x5d, 0x9b, 0xe8, + 0xc8, 0xa5, 0x44, 0x91, 0xfb, 0x08, 0xde, 0x32, 0x2f, 0x4c, 0xc4, 0x13, 0x0b, 0x7c, 0x0b, 0x90, + 0x17, 0x62, 0x03, 0xcf, 0x87, 0x2e, 0x8c, 0x6d, 0xd6, 0xb2, 0xce, 0xea, 0xa4, 0xb3, 0x79, 0x00, + 0xcd, 0x75, 0xd5, 0x24, 0x65, 0xba, 0x4f, 0xf7, 0x4a, 0x22, 0x63, 0x51, 0xb4, 0x14, 0x69, 0xa5, + 0x62, 0x8b, 0xcd, 0xec, 0x8b, 0x18, 0xaa, 0x0a, 0x13, 0xc4, 0x51, 0xdf, 0xe2, 0x4b, 0x23, 0x82, + 0x98, 0xe2, 0x5b, 0xc6, 0xe0, 0x8b, 0x85, 0x1f, 0x4b, 0xf4, 0x2f, 0x11, 0x11, 0x84, 0xfb, 0xba, + 0xbb, 0xbb, 0x31, 0x50, 0x4d, 0x27, 0x0b, 0x0b, 0x14, 0xf1, 0x10, 0x2c, 0x70, 0x7c, 0xc1, 0x82, + 0x06, 0xf5, 0x50, 0x2f, 0x14, 0x25, 0xbd, 0x01, 0x4a, 0xea, 0xc6, 0xb6, 0x97, 0x26, 0x23, 0x8a, + 0x55, 0x76, 0xa1, 0xee, 0xe9, 0x2b, 0x13, 0xee, 0x8b, 0x69, 0x9c, 0xfa, 0x38, 0x29, 0xe6, 0x96, + 0x8c, 0x1a, 0x34, 0xb7, 0x78, 0xab, 0x07, 0x65, 0x6b, 0x11, 0x8e, 0x33, 0xb7, 0xf4, 0xfe, 0x11, + 0xf3, 0x27, 0x91, 0xb6, 0x6a, 0xb2, 0xb3, 0x4a, 0xec, 0x12, 0x27, 0xfa, 0x93, 0x94, 0x2e, 0xad, + 0x7c, 0x0c, 0xdd, 0x87, 0x12, 0xab, 0xf0, 0xd0, 0xf3, 0xe3, 0x8b, 0x3f, 0x4e, 0xfd, 0xe2, 0x04, + 0x2c, 0x41, 0x78, 0x0f, 0x4e, 0xa6, 0x94, 0x7e, 0x89, 0x71, 0x6e, 0x7c, 0x99, 0x38, 0xc9, 0xca, + 0x55, 0x40, 0xf1, 0xbf, 0x1c, 0x24, 0x1e, 0x53, 0xea, 0x3f, 0x13, 0x32, 0x2c, 0x11, 0xff, 0xd7, + 0x40, 0xe2, 0x12, 0xa9, 0x7f, 0x2e, 0x98, 0xb4, 0xc4, 0x3d, 0x00, 0xbf, 0xc0, 0x4b, 0x3c, 0x8f, + 0x58, 0xfd, 0x37, 0x81, 0xe4, 0xf2, 0x7f, 0x4a, 0x50, 0xf6, 0x5e, 0xdc, 0x3c, 0x83, 0xe4, 0xfe, + 0x19, 0x64, 0xdb, 0x1f, 0xc2, 0x6c, 0xe4, 0xe9, 0x7e, 0xa2, 0xf1, 0x24, 0x3f, 0xef, 0x9f, 0x74, + 0x42, 0xf7, 0xf9, 0x9f, 0xa5, 0x45, 0xe0, 0x7d, 0x21, 0x2d, 0x63, 0x8f, 0xc6, 0xdc, 0x09, 0x84, + 0x9f, 0x7a, 0x84, 0xbd, 0x0b, 0x10, 0x88, 0x80, 0xe3, 0xaf, 0x41, 0x89, 0x53, 0x9f, 0xc4, 0xf0, + 0xfa, 0x21, 0xfd, 0xc6, 0x04, 0x72, 0x0e, 0xb1, 0xae, 0x68, 0x3b, 0x25, 0xc5, 0xba, 0x52, 0x9a, + 0x38, 0x89, 0x7e, 0x36, 0xbd, 0x47, 0xf3, 0x54, 0xec, 0xed, 0xe6, 0xf5, 0x6f, 0xbf, 0xda, 0xd7, + 0xdd, 0xdd, 0xd1, 0x36, 0xf9, 0x72, 0x8d, 0xa1, 0xbe, 0xac, 0x5b, 0xfc, 0xd7, 0x35, 0x4f, 0xd1, + 0xaf, 0xd1, 0xd9, 0xd7, 0xc8, 0x1a, 0xc3, 0xed, 0xed, 0x12, 0x1d, 0x5d, 0xff, 0x5f, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x49, 0xff, 0x1b, 0x78, 0x8f, 0x40, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/querycoord/global_meta_broker.go b/internal/querycoord/global_meta_broker.go index b2dc51d466..bbaa5e2a6f 100644 --- a/internal/querycoord/global_meta_broker.go +++ b/internal/querycoord/global_meta_broker.go @@ -479,10 +479,11 @@ func (broker *globalMetaBroker) getSegmentStates(ctx context.Context, segmentID return resp.States[0], nil } -func (broker *globalMetaBroker) acquireSegmentsReferLock(ctx context.Context, segmentIDs []UniqueID) error { +func (broker *globalMetaBroker) acquireSegmentsReferLock(ctx context.Context, taskID int64, segmentIDs []UniqueID) error { ctx, cancel := context.WithTimeout(ctx, timeoutForRPC) defer cancel() acquireSegLockReq := &datapb.AcquireSegmentLockRequest{ + TaskID: taskID, SegmentIDs: segmentIDs, NodeID: Params.QueryCoordCfg.GetNodeID(), } @@ -501,13 +502,13 @@ func (broker *globalMetaBroker) acquireSegmentsReferLock(ctx context.Context, se return nil } -func (broker *globalMetaBroker) releaseSegmentReferLock(ctx context.Context, segmentIDs []UniqueID) error { +func (broker *globalMetaBroker) releaseSegmentReferLock(ctx context.Context, taskID int64, segmentIDs []UniqueID) error { ctx, cancel := context.WithTimeout(ctx, timeoutForRPC) defer cancel() releaseSegReferLockReq := &datapb.ReleaseSegmentLockRequest{ - SegmentIDs: segmentIDs, - NodeID: Params.QueryCoordCfg.GetNodeID(), + TaskID: taskID, + NodeID: Params.QueryCoordCfg.GetNodeID(), } if err := retry.Do(ctx, func() error { diff --git a/internal/querycoord/task.go b/internal/querycoord/task.go index fc42b7e3d3..518b6f78bd 100644 --- a/internal/querycoord/task.go +++ b/internal/querycoord/task.go @@ -1239,7 +1239,7 @@ func (lst *loadSegmentTask) preExecute(ctx context.Context) error { zap.Int64("loaded nodeID", lst.DstNodeID), zap.Int64("taskID", lst.getTaskID())) - if err := lst.broker.acquireSegmentsReferLock(ctx, segmentIDs); err != nil { + if err := lst.broker.acquireSegmentsReferLock(ctx, lst.taskID, segmentIDs); err != nil { log.Error("acquire reference lock on segments failed", zap.Int64s("segmentIDs", segmentIDs), zap.Error(err)) return err @@ -1267,7 +1267,7 @@ func (lst *loadSegmentTask) postExecute(context.Context) error { for _, info := range lst.Infos { segmentIDs = append(segmentIDs, info.SegmentID) } - if err := lst.broker.releaseSegmentReferLock(lst.ctx, segmentIDs); err != nil { + if err := lst.broker.releaseSegmentReferLock(lst.ctx, lst.taskID, segmentIDs); err != nil { panic(err) } diff --git a/internal/rootcoord/import_manager.go b/internal/rootcoord/import_manager.go index 73a0860099..996b8e10c8 100644 --- a/internal/rootcoord/import_manager.go +++ b/internal/rootcoord/import_manager.go @@ -135,7 +135,7 @@ func (m *importManager) sendOutTasksLoop(wg *sync.WaitGroup) { } // expireOldTasksLoop starts a loop that checks and expires old tasks every `ImportTaskExpiration` seconds. -func (m *importManager) expireOldTasksLoop(wg *sync.WaitGroup, releaseLockFunc func(context.Context, []int64) error) { +func (m *importManager) expireOldTasksLoop(wg *sync.WaitGroup, releaseLockFunc func(context.Context, int64, []int64) error) { defer wg.Done() ticker := time.NewTicker(time.Duration(expireOldTasksInterval) * time.Millisecond) defer ticker.Stop() @@ -607,7 +607,7 @@ func (m *importManager) updateImportTaskStore(ti *datapb.ImportTaskInfo) error { } // expireOldTasks marks expires tasks as failed. -func (m *importManager) expireOldTasks(releaseLockFunc func(context.Context, []int64) error) { +func (m *importManager) expireOldTasks(releaseLockFunc func(context.Context, int64, []int64) error) { // Expire old pending tasks, if any. func() { m.pendingLock.Lock() @@ -622,7 +622,7 @@ func (m *importManager) expireOldTasks(releaseLockFunc func(context.Context, []i log.Info("releasing seg ref locks on expired import task", zap.Int64s("segment IDs", t.GetState().GetSegments())) err := retry.Do(m.ctx, func() error { - return releaseLockFunc(m.ctx, t.GetState().GetSegments()) + return releaseLockFunc(m.ctx, t.GetId(), t.GetState().GetSegments()) }, retry.Attempts(100)) if err != nil { log.Error("failed to release lock, about to panic!") @@ -646,7 +646,7 @@ func (m *importManager) expireOldTasks(releaseLockFunc func(context.Context, []i log.Info("releasing seg ref locks on expired import task", zap.Int64s("segment IDs", v.GetState().GetSegments())) err := retry.Do(m.ctx, func() error { - return releaseLockFunc(m.ctx, v.GetState().GetSegments()) + return releaseLockFunc(m.ctx, v.GetId(), v.GetState().GetSegments()) }, retry.Attempts(100)) if err != nil { log.Error("failed to release lock, about to panic!") diff --git a/internal/rootcoord/import_manager_test.go b/internal/rootcoord/import_manager_test.go index 7e08f68d50..ac034036be 100644 --- a/internal/rootcoord/import_manager_test.go +++ b/internal/rootcoord/import_manager_test.go @@ -93,7 +93,7 @@ func TestImportManager_NewImportManager(t *testing.T) { mgr.init(ctx) var wgLoop sync.WaitGroup wgLoop.Add(2) - mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64s []int64) error { + mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64 int64, int64s []int64) error { return nil }) mgr.sendOutTasksLoop(&wgLoop) @@ -110,7 +110,7 @@ func TestImportManager_NewImportManager(t *testing.T) { mgr.init(context.TODO()) var wgLoop sync.WaitGroup wgLoop.Add(2) - mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64s []int64) error { + mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64 int64, int64s []int64) error { return nil }) mgr.sendOutTasksLoop(&wgLoop) @@ -134,7 +134,7 @@ func TestImportManager_NewImportManager(t *testing.T) { mgr.loadFromTaskStore() var wgLoop sync.WaitGroup wgLoop.Add(2) - mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64s []int64) error { + mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64 int64, int64s []int64) error { return nil }) mgr.sendOutTasksLoop(&wgLoop) @@ -151,7 +151,7 @@ func TestImportManager_NewImportManager(t *testing.T) { mgr.init(ctx) var wgLoop sync.WaitGroup wgLoop.Add(2) - mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64s []int64) error { + mgr.expireOldTasksLoop(&wgLoop, func(ctx context.Context, int64 int64, int64s []int64) error { return nil }) mgr.sendOutTasksLoop(&wgLoop) diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index c5c6dc3831..0f6453a029 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -155,10 +155,10 @@ type Core struct { CallFlushOnCollection func(ctx context.Context, cID int64, segIDs []int64) error // CallAddSegRefLock triggers AcquireSegmentLock method on DataCoord. - CallAddSegRefLock func(ctx context.Context, segIDs []int64) (retErr error) + CallAddSegRefLock func(ctx context.Context, taskID int64, segIDs []int64) (retErr error) // CallReleaseSegRefLock triggers ReleaseSegmentLock method on DataCoord. - CallReleaseSegRefLock func(ctx context.Context, segIDs []int64) (retErr error) + CallReleaseSegRefLock func(ctx context.Context, taskID int64, segIDs []int64) (retErr error) //Proxy manager proxyManager *proxyManager @@ -697,7 +697,7 @@ func (c *Core) SetDataCoord(ctx context.Context, s types.DataCoord) error { return nil } - c.CallAddSegRefLock = func(ctx context.Context, segIDs []int64) (retErr error) { + c.CallAddSegRefLock = func(ctx context.Context, taskID int64, segIDs []int64) (retErr error) { defer func() { if err := recover(); err != nil { retErr = fmt.Errorf("add seg ref lock panic, msg = %v", err) @@ -710,6 +710,7 @@ func (c *Core) SetDataCoord(ctx context.Context, s types.DataCoord) error { resp, _ := s.AcquireSegmentLock(ctx, &datapb.AcquireSegmentLockRequest{ SegmentIDs: segIDs, NodeID: c.session.ServerID, + TaskID: taskID, }) if resp.GetErrorCode() != commonpb.ErrorCode_Success { return fmt.Errorf("failed to acquire segment lock %s", resp.GetReason()) @@ -720,7 +721,7 @@ func (c *Core) SetDataCoord(ctx context.Context, s types.DataCoord) error { return nil } - c.CallReleaseSegRefLock = func(ctx context.Context, segIDs []int64) (retErr error) { + c.CallReleaseSegRefLock = func(ctx context.Context, taskID int64, segIDs []int64) (retErr error) { defer func() { if err := recover(); err != nil { retErr = fmt.Errorf("release seg ref lock panic, msg = %v", err) @@ -733,6 +734,7 @@ func (c *Core) SetDataCoord(ctx context.Context, s types.DataCoord) error { resp, _ := s.ReleaseSegmentLock(ctx, &datapb.ReleaseSegmentLockRequest{ SegmentIDs: segIDs, NodeID: c.session.ServerID, + TaskID: taskID, }) if resp.GetErrorCode() != commonpb.ErrorCode_Success { return fmt.Errorf("failed to release segment lock %s", resp.GetReason()) @@ -2409,7 +2411,7 @@ func (c *Core) ReportImport(ctx context.Context, ir *rootcoordpb.ImportResult) ( if ir.GetState() == commonpb.ImportState_ImportAllocSegment { // Lock the segments, so we don't lose track of them when compaction happens. // Note that these locks will be unlocked in c.postImportPersistLoop() -> checkSegmentLoadedLoop(). - if err := c.CallAddSegRefLock(ctx, ir.GetSegments()); err != nil { + if err := c.CallAddSegRefLock(ctx, ir.GetTaskId(), ir.GetSegments()); err != nil { log.Error("failed to acquire segment ref lock", zap.Error(err)) return &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -2450,7 +2452,7 @@ func (c *Core) ReportImport(ctx context.Context, ir *rootcoordpb.ImportResult) ( // Release segments when task fails. log.Info("task failed, release segment ref locks") err := retry.Do(ctx, func() error { - return c.CallReleaseSegRefLock(ctx, ir.GetSegments()) + return c.CallReleaseSegRefLock(ctx, ir.GetTaskId(), ir.GetSegments()) }, retry.Attempts(100)) if err != nil { log.Error("failed to release lock, about to panic!") @@ -2630,7 +2632,7 @@ func (c *Core) checkSegmentLoadedLoop(ctx context.Context, taskID int64, colID i defer func() { log.Info("we are done checking segment loading state, release segment ref locks") err := retry.Do(ctx, func() error { - return c.CallReleaseSegRefLock(ctx, segIDs) + return c.CallReleaseSegRefLock(ctx, taskID, segIDs) }, retry.Attempts(100)) if err != nil { log.Error("failed to release lock, about to panic!") diff --git a/internal/rootcoord/root_coord_test.go b/internal/rootcoord/root_coord_test.go index 7491f44f81..853f694aa1 100644 --- a/internal/rootcoord/root_coord_test.go +++ b/internal/rootcoord/root_coord_test.go @@ -3074,13 +3074,13 @@ func TestCheckInit(t *testing.T) { err = c.checkInit() assert.Error(t, err) - c.CallAddSegRefLock = func(context.Context, []int64) error { + c.CallAddSegRefLock = func(context.Context, int64, []int64) error { return nil } err = c.checkInit() assert.Error(t, err) - c.CallReleaseSegRefLock = func(context.Context, []int64) error { + c.CallReleaseSegRefLock = func(context.Context, int64, []int64) error { return nil } err = c.checkInit()