From 7fee4743f50376483fa11d22f8d59200a740608c Mon Sep 17 00:00:00 2001 From: bigsheeper Date: Thu, 24 Mar 2022 20:17:25 +0800 Subject: [PATCH] Improve log print in OOM error info (#16170) Signed-off-by: bigsheeper --- internal/querynode/segment_loader.go | 16 ++++++++++------ internal/querynode/segment_loader_test.go | 14 ++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/internal/querynode/segment_loader.go b/internal/querynode/segment_loader.go index 7094b093f0..bcd27fd2cc 100644 --- a/internal/querynode/segment_loader.go +++ b/internal/querynode/segment_loader.go @@ -665,20 +665,24 @@ func (loader *segmentLoader) checkSegmentSize(collectionID UniqueID, segmentLoad return fmt.Errorf("get memory failed when checkSegmentSize, collectionID = %d", collectionID) } - usedMemAfterLoad := int64(usedMem) - maxSegmentSize := int64(0) + usedMemAfterLoad := usedMem + maxSegmentSize := uint64(0) for _, loadInfo := range segmentLoadInfos { - segmentSize := loadInfo.SegmentSize + segmentSize := uint64(loadInfo.SegmentSize) usedMemAfterLoad += segmentSize if segmentSize > maxSegmentSize { maxSegmentSize = segmentSize } } + toMB := func(mem uint64) float64 { + return float64(mem) / 1024 / 1024 + } + // when load segment, data will be copied from go memory to c++ memory - if uint64(usedMemAfterLoad+maxSegmentSize*int64(concurrency)) > uint64(float64(totalMem)*Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) { - return fmt.Errorf("load segment failed, OOM if load, collectionID = %d, maxSegmentSize = %d, usedMemAfterLoad = %d, totalMem = %d, thresholdFactor = %f", - collectionID, maxSegmentSize, usedMemAfterLoad, totalMem, Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) + if usedMemAfterLoad+maxSegmentSize*uint64(concurrency) > uint64(float64(totalMem)*Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) { + return fmt.Errorf("load segment failed, OOM if load, collectionID = %d, maxSegmentSize = %.2f MB, concurrency = %d, usedMemAfterLoad = %.2f MB, totalMem = %.2f MB, thresholdFactor = %f", + collectionID, toMB(maxSegmentSize), concurrency, toMB(usedMemAfterLoad), toMB(totalMem), Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) } return nil diff --git a/internal/querynode/segment_loader_test.go b/internal/querynode/segment_loader_test.go index 7116a7721f..67d700e30b 100644 --- a/internal/querynode/segment_loader_test.go +++ b/internal/querynode/segment_loader_test.go @@ -23,13 +23,12 @@ import ( "runtime" "testing" - "github.com/milvus-io/milvus/internal/mq/msgstream" - "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/assert" - + "github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/util/funcutil" @@ -321,13 +320,8 @@ func TestSegmentLoader_checkSegmentSize(t *testing.T) { loader := node.loader assert.NotNil(t, loader) - err = loader.checkSegmentSize(defaultSegmentID, []*querypb.SegmentLoadInfo{{SegmentID: defaultSegmentID, SegmentSize: 1024}}, runtime.GOMAXPROCS(0)) + err = loader.checkSegmentSize(defaultCollectionID, []*querypb.SegmentLoadInfo{{SegmentID: defaultSegmentID, SegmentSize: 1024}}, runtime.GOMAXPROCS(0)) assert.NoError(t, err) - - //totalMem, err := getTotalMemory() - //assert.NoError(t, err) - //err = historical.loader.checkSegmentSize(defaultSegmentID, map[UniqueID]int64{defaultSegmentID: int64(totalMem * 2)}) - //assert.Error(t, err) } func TestSegmentLoader_testLoadGrowing(t *testing.T) {