mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-08 10:08:42 +08:00
fix: Correct flush buffer size metrics (#29571)
See also: #29204 Signed-off-by: yangxuan <xuan.yang@zilliz.com>
This commit is contained in:
parent
bebe1baf18
commit
a3aff37f73
@ -24,13 +24,15 @@ import (
|
|||||||
"github.com/milvus-io/milvus/internal/storage"
|
"github.com/milvus-io/milvus/internal/storage"
|
||||||
"github.com/milvus-io/milvus/internal/util/typeutil"
|
"github.com/milvus-io/milvus/internal/util/typeutil"
|
||||||
"github.com/milvus-io/milvus/pkg/common"
|
"github.com/milvus-io/milvus/pkg/common"
|
||||||
|
"github.com/milvus-io/milvus/pkg/metrics"
|
||||||
"github.com/milvus-io/milvus/pkg/mq/msgstream"
|
"github.com/milvus-io/milvus/pkg/mq/msgstream"
|
||||||
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
||||||
|
"github.com/milvus-io/milvus/pkg/util/testutils"
|
||||||
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BFWriteBufferSuite struct {
|
type BFWriteBufferSuite struct {
|
||||||
suite.Suite
|
testutils.PromMetricsSuite
|
||||||
collID int64
|
collID int64
|
||||||
channelName string
|
channelName string
|
||||||
collSchema *schemapb.CollectionSchema
|
collSchema *schemapb.CollectionSchema
|
||||||
@ -173,8 +175,13 @@ func (s *BFWriteBufferSuite) TestBufferData() {
|
|||||||
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
||||||
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
||||||
|
|
||||||
|
metrics.DataNodeFlowGraphBufferDataSize.Reset()
|
||||||
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
|
value, err := metrics.DataNodeFlowGraphBufferDataSize.GetMetricWithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(s.metacache.Collection()))
|
||||||
|
s.NoError(err)
|
||||||
|
s.MetricsEqual(value, 5524)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *BFWriteBufferSuite) TestAutoSync() {
|
func (s *BFWriteBufferSuite) TestAutoSync() {
|
||||||
@ -191,9 +198,11 @@ func (s *BFWriteBufferSuite) TestAutoSync() {
|
|||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
||||||
|
seg1 := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1002}, metacache.NewBloomFilterSet())
|
||||||
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg})
|
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg})
|
||||||
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false)
|
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false).Once()
|
||||||
s.metacache.EXPECT().GetSegmentByID(int64(1002)).Return(seg, true)
|
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(seg, true).Once()
|
||||||
|
s.metacache.EXPECT().GetSegmentByID(int64(1002)).Return(seg1, true)
|
||||||
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything).Return([]int64{1002})
|
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything).Return([]int64{1002})
|
||||||
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{})
|
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{})
|
||||||
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything).Return()
|
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything).Return()
|
||||||
@ -204,8 +213,13 @@ func (s *BFWriteBufferSuite) TestAutoSync() {
|
|||||||
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
||||||
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
||||||
|
|
||||||
|
metrics.DataNodeFlowGraphBufferDataSize.Reset()
|
||||||
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
|
value, err := metrics.DataNodeFlowGraphBufferDataSize.GetMetricWithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(s.metacache.Collection()))
|
||||||
|
s.NoError(err)
|
||||||
|
s.MetricsEqual(value, 0)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -265,12 +279,14 @@ func (s *BFWriteBufferSuite) TestAutoSyncWithStorageV2() {
|
|||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
||||||
|
seg1 := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1002}, metacache.NewBloomFilterSet())
|
||||||
segCompacted := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
segCompacted := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
||||||
metacache.CompactTo(2001)(segCompacted)
|
metacache.CompactTo(2001)(segCompacted)
|
||||||
|
|
||||||
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg, segCompacted})
|
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg, segCompacted})
|
||||||
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false)
|
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false).Once()
|
||||||
s.metacache.EXPECT().GetSegmentByID(int64(1002)).Return(seg, true)
|
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(seg, true).Once()
|
||||||
|
s.metacache.EXPECT().GetSegmentByID(int64(1002)).Return(seg1, true)
|
||||||
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything).Return([]int64{1002})
|
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything).Return([]int64{1002})
|
||||||
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{1003}) // mocked compacted
|
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{1003}) // mocked compacted
|
||||||
s.metacache.EXPECT().RemoveSegments(mock.Anything).Return([]int64{1003})
|
s.metacache.EXPECT().RemoveSegments(mock.Anything).Return([]int64{1003})
|
||||||
@ -282,8 +298,13 @@ func (s *BFWriteBufferSuite) TestAutoSyncWithStorageV2() {
|
|||||||
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
pks, msg := s.composeInsertMsg(1000, 10, 128)
|
||||||
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
delMsg := s.composeDeleteMsg(lo.Map(pks, func(id int64, _ int) storage.PrimaryKey { return storage.NewInt64PrimaryKey(id) }))
|
||||||
|
|
||||||
|
metrics.DataNodeFlowGraphBufferDataSize.Reset()
|
||||||
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
|
value, err := metrics.DataNodeFlowGraphBufferDataSize.GetMetricWithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(s.metacache.Collection()))
|
||||||
|
s.NoError(err)
|
||||||
|
s.MetricsEqual(value, 0)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -118,21 +118,22 @@ func (ib *InsertBuffer) Yield() *storage.InsertData {
|
|||||||
return ib.buffer
|
return ib.buffer
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ib *InsertBuffer) Buffer(msgs []*msgstream.InsertMsg, startPos, endPos *msgpb.MsgPosition) ([]storage.FieldData, error) {
|
func (ib *InsertBuffer) Buffer(msgs []*msgstream.InsertMsg, startPos, endPos *msgpb.MsgPosition) ([]storage.FieldData, int64, error) {
|
||||||
pkData := make([]storage.FieldData, 0, len(msgs))
|
pkData := make([]storage.FieldData, 0, len(msgs))
|
||||||
|
var totalMemSize int64 = 0
|
||||||
for _, msg := range msgs {
|
for _, msg := range msgs {
|
||||||
tmpBuffer, err := storage.InsertMsgToInsertData(msg, ib.collSchema)
|
tmpBuffer, err := storage.InsertMsgToInsertData(msg, ib.collSchema)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("failed to transfer insert msg to insert data", zap.Error(err))
|
log.Warn("failed to transfer insert msg to insert data", zap.Error(err))
|
||||||
return nil, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
pkFieldData, err := storage.GetPkFromInsertData(ib.collSchema, tmpBuffer)
|
pkFieldData, err := storage.GetPkFromInsertData(ib.collSchema, tmpBuffer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
if pkFieldData.RowNum() != tmpBuffer.GetRowNum() {
|
if pkFieldData.RowNum() != tmpBuffer.GetRowNum() {
|
||||||
return nil, merr.WrapErrServiceInternal("pk column row num not match")
|
return nil, 0, merr.WrapErrServiceInternal("pk column row num not match")
|
||||||
}
|
}
|
||||||
pkData = append(pkData, pkFieldData)
|
pkData = append(pkData, pkFieldData)
|
||||||
|
|
||||||
@ -141,13 +142,14 @@ func (ib *InsertBuffer) Buffer(msgs []*msgstream.InsertMsg, startPos, endPos *ms
|
|||||||
tsData, err := storage.GetTimestampFromInsertData(tmpBuffer)
|
tsData, err := storage.GetTimestampFromInsertData(tmpBuffer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("no timestamp field found in insert msg", zap.Error(err))
|
log.Warn("no timestamp field found in insert msg", zap.Error(err))
|
||||||
return nil, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// update buffer size
|
// update buffer size
|
||||||
ib.UpdateStatistics(int64(tmpBuffer.GetRowNum()), int64(tmpBuffer.GetMemorySize()), ib.getTimestampRange(tsData), startPos, endPos)
|
ib.UpdateStatistics(int64(tmpBuffer.GetRowNum()), int64(tmpBuffer.GetMemorySize()), ib.getTimestampRange(tsData), startPos, endPos)
|
||||||
|
totalMemSize += int64(tmpBuffer.GetMemorySize())
|
||||||
}
|
}
|
||||||
return pkData, nil
|
return pkData, totalMemSize, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ib *InsertBuffer) getTimestampRange(tsData *storage.Int64FieldData) TimeRange {
|
func (ib *InsertBuffer) getTimestampRange(tsData *storage.Int64FieldData) TimeRange {
|
||||||
|
|||||||
@ -134,7 +134,7 @@ func (s *InsertBufferSuite) TestBuffer() {
|
|||||||
insertBuffer, err := NewInsertBuffer(s.collSchema)
|
insertBuffer, err := NewInsertBuffer(s.collSchema)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
fieldData, err := insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
fieldData, memSize, err := insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
pkData := lo.Map(fieldData, func(fd storage.FieldData, _ int) []int64 {
|
pkData := lo.Map(fieldData, func(fd storage.FieldData, _ int) []int64 {
|
||||||
@ -142,6 +142,7 @@ func (s *InsertBufferSuite) TestBuffer() {
|
|||||||
})
|
})
|
||||||
s.ElementsMatch(pks, lo.Flatten(pkData))
|
s.ElementsMatch(pks, lo.Flatten(pkData))
|
||||||
s.EqualValues(100, insertBuffer.MinTimestamp())
|
s.EqualValues(100, insertBuffer.MinTimestamp())
|
||||||
|
s.EqualValues(5364, memSize)
|
||||||
})
|
})
|
||||||
|
|
||||||
s.Run("pk_not_found", func() {
|
s.Run("pk_not_found", func() {
|
||||||
@ -152,7 +153,7 @@ func (s *InsertBufferSuite) TestBuffer() {
|
|||||||
insertBuffer, err := NewInsertBuffer(s.collSchema)
|
insertBuffer, err := NewInsertBuffer(s.collSchema)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
_, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
_, _, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.Error(err)
|
s.Error(err)
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -180,7 +181,7 @@ func (s *InsertBufferSuite) TestBuffer() {
|
|||||||
insertBuffer, err := NewInsertBuffer(badSchema)
|
insertBuffer, err := NewInsertBuffer(badSchema)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
_, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
_, _, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.Error(err)
|
s.Error(err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -196,7 +197,7 @@ func (s *InsertBufferSuite) TestYield() {
|
|||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
pks, insertMsg := s.composeInsertMsg(10, 128)
|
pks, insertMsg := s.composeInsertMsg(10, 128)
|
||||||
_, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
_, _, err = insertBuffer.Buffer([]*msgstream.InsertMsg{insertMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
result = insertBuffer.Yield()
|
result = insertBuffer.Yield()
|
||||||
|
|||||||
@ -118,6 +118,12 @@ func (wb *l0WriteBuffer) getL0SegmentID(partitionID int64, startPos *msgpb.MsgPo
|
|||||||
State: commonpb.SegmentState_Growing,
|
State: commonpb.SegmentState_Growing,
|
||||||
Level: datapb.SegmentLevel_L0,
|
Level: datapb.SegmentLevel_L0,
|
||||||
}, func(_ *datapb.SegmentInfo) *metacache.BloomFilterSet { return metacache.NewBloomFilterSet() }, metacache.SetStartPosRecorded(false))
|
}, func(_ *datapb.SegmentInfo) *metacache.BloomFilterSet { return metacache.NewBloomFilterSet() }, metacache.SetStartPosRecorded(false))
|
||||||
|
log.Info("Add a new level zero segment",
|
||||||
|
zap.Int64("segmentID", segmentID),
|
||||||
|
zap.String("level", datapb.SegmentLevel_L0.String()),
|
||||||
|
zap.String("channel", wb.channelName),
|
||||||
|
zap.Any("start position", startPos),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
return segmentID
|
return segmentID
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package writebuffer
|
package writebuffer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -18,13 +19,15 @@ import (
|
|||||||
"github.com/milvus-io/milvus/internal/proto/datapb"
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
||||||
"github.com/milvus-io/milvus/internal/storage"
|
"github.com/milvus-io/milvus/internal/storage"
|
||||||
"github.com/milvus-io/milvus/pkg/common"
|
"github.com/milvus-io/milvus/pkg/common"
|
||||||
|
"github.com/milvus-io/milvus/pkg/metrics"
|
||||||
"github.com/milvus-io/milvus/pkg/mq/msgstream"
|
"github.com/milvus-io/milvus/pkg/mq/msgstream"
|
||||||
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
||||||
|
"github.com/milvus-io/milvus/pkg/util/testutils"
|
||||||
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
type L0WriteBufferSuite struct {
|
type L0WriteBufferSuite struct {
|
||||||
suite.Suite
|
testutils.PromMetricsSuite
|
||||||
channelName string
|
channelName string
|
||||||
collID int64
|
collID int64
|
||||||
collSchema *schemapb.CollectionSchema
|
collSchema *schemapb.CollectionSchema
|
||||||
@ -161,13 +164,18 @@ func (s *L0WriteBufferSuite) TestBufferData() {
|
|||||||
|
|
||||||
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ID: 1000}, metacache.NewBloomFilterSet())
|
||||||
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg})
|
s.metacache.EXPECT().GetSegmentsBy(mock.Anything, mock.Anything).Return([]*metacache.SegmentInfo{seg})
|
||||||
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false)
|
s.metacache.EXPECT().GetSegmentByID(int64(1000)).Return(nil, false).Once()
|
||||||
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything).Return()
|
s.metacache.EXPECT().AddSegment(mock.Anything, mock.Anything, mock.Anything).Return()
|
||||||
s.metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
s.metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything).Return()
|
||||||
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{})
|
s.metacache.EXPECT().GetSegmentIDsBy(mock.Anything, mock.Anything).Return([]int64{})
|
||||||
|
|
||||||
|
metrics.DataNodeFlowGraphBufferDataSize.Reset()
|
||||||
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
err = wb.BufferData([]*msgstream.InsertMsg{msg}, []*msgstream.DeleteMsg{delMsg}, &msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 200})
|
||||||
s.NoError(err)
|
s.NoError(err)
|
||||||
|
|
||||||
|
value, err := metrics.DataNodeFlowGraphBufferDataSize.GetMetricWithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(s.metacache.Collection()))
|
||||||
|
s.NoError(err)
|
||||||
|
s.MetricsEqual(value, 5524)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *L0WriteBufferSuite) TestCreateFailure() {
|
func (s *L0WriteBufferSuite) TestCreateFailure() {
|
||||||
|
|||||||
@ -339,7 +339,7 @@ func (wb *writeBufferBase) bufferInsert(insertMsgs []*msgstream.InsertMsg, start
|
|||||||
|
|
||||||
segBuf := wb.getOrCreateBuffer(segmentID)
|
segBuf := wb.getOrCreateBuffer(segmentID)
|
||||||
|
|
||||||
pkData, err := segBuf.insertBuffer.Buffer(msgs, startPos, endPos)
|
pkData, totalMemSize, err := segBuf.insertBuffer.Buffer(msgs, startPos, endPos)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("failed to buffer insert data", zap.Int64("segmentID", segmentID), zap.Error(err))
|
log.Warn("failed to buffer insert data", zap.Int64("segmentID", segmentID), zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -348,10 +348,7 @@ func (wb *writeBufferBase) bufferInsert(insertMsgs []*msgstream.InsertMsg, start
|
|||||||
wb.metaCache.UpdateSegments(metacache.UpdateBufferedRows(segBuf.insertBuffer.rows),
|
wb.metaCache.UpdateSegments(metacache.UpdateBufferedRows(segBuf.insertBuffer.rows),
|
||||||
metacache.WithSegmentIDs(segmentID))
|
metacache.WithSegmentIDs(segmentID))
|
||||||
|
|
||||||
totalSize := lo.SumBy(pkData, func(iData storage.FieldData) float64 {
|
metrics.DataNodeFlowGraphBufferDataSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(wb.collectionID)).Add(float64(totalMemSize))
|
||||||
return float64(iData.GetMemorySize())
|
|
||||||
})
|
|
||||||
metrics.DataNodeFlowGraphBufferDataSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(wb.collectionID)).Add(totalSize)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return segmentPKData, nil
|
return segmentPKData, nil
|
||||||
@ -375,7 +372,7 @@ func (wb *writeBufferBase) getSyncTask(ctx context.Context, segmentID int64) (sy
|
|||||||
return nil, merr.WrapErrSegmentNotFound(segmentID)
|
return nil, merr.WrapErrSegmentNotFound(segmentID)
|
||||||
}
|
}
|
||||||
var batchSize int64
|
var batchSize int64
|
||||||
var totalMemSize float64
|
var totalMemSize float64 = 0
|
||||||
var tsFrom, tsTo uint64
|
var tsFrom, tsTo uint64
|
||||||
|
|
||||||
insert, delta, timeRange, startPos := wb.yieldBuffer(segmentID)
|
insert, delta, timeRange, startPos := wb.yieldBuffer(segmentID)
|
||||||
@ -391,6 +388,7 @@ func (wb *writeBufferBase) getSyncTask(ctx context.Context, segmentID int64) (sy
|
|||||||
if delta != nil {
|
if delta != nil {
|
||||||
totalMemSize += float64(delta.Size())
|
totalMemSize += float64(delta.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
actions = append(actions, metacache.StartSyncing(batchSize))
|
actions = append(actions, metacache.StartSyncing(batchSize))
|
||||||
wb.metaCache.UpdateSegments(metacache.MergeSegmentAction(actions...), metacache.WithSegmentIDs(segmentID))
|
wb.metaCache.UpdateSegments(metacache.MergeSegmentAction(actions...), metacache.WithSegmentIDs(segmentID))
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user