From 36bb87d9b61419a546dd744f287d37f4905ea492 Mon Sep 17 00:00:00 2001 From: "yihao.dai" Date: Thu, 7 Nov 2024 17:12:25 +0800 Subject: [PATCH] fix: Fix large growing segment (#37388) Consider the `sealProportion` factor during segment allocation. issue: https://github.com/milvus-io/milvus/issues/37387 Signed-off-by: bigsheeper --- internal/datacoord/segment_allocation_policy.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/datacoord/segment_allocation_policy.go b/internal/datacoord/segment_allocation_policy.go index 380183146d..58b1ff5c40 100644 --- a/internal/datacoord/segment_allocation_policy.go +++ b/internal/datacoord/segment_allocation_policy.go @@ -108,7 +108,18 @@ func AllocatePolicyL1(segments []*SegmentInfo, count int64, for _, allocation := range segment.allocations { allocSize += allocation.NumOfRows } - free := segment.GetMaxRowNum() - segment.GetNumOfRows() - allocSize + + // When inserts are too fast, hardTimeTick may lag, causing segment to be unable to seal in time. + // To prevent allocating large segment, introducing the sealProportion factor here. + // The condition `free < 0` ensures that the allocation exceeds the minimum sealable size, + // preventing segments from remaining unsealable indefinitely. + maxRowsWithSealProportion := int64(float64(segment.GetMaxRowNum()) * paramtable.Get().DataCoordCfg.SegmentSealProportion.GetAsFloat()) + free := maxRowsWithSealProportion - segment.GetNumOfRows() - allocSize + if free < 0 { + continue + } + + free = segment.GetMaxRowNum() - segment.GetNumOfRows() - allocSize if free < count { continue }