diff --git a/configs/milvus.yaml b/configs/milvus.yaml index a9881e6944..0b01da6594 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -1139,7 +1139,7 @@ quotaAndLimits: # DML rates would be reduced according to the ratio of time tick delay to maxTimeTickDelay, # if time tick delay is greater than maxTimeTickDelay, all DML requests would be rejected. # seconds - maxTimeTickDelay: 300 + maxTimeTickDelay: 1200 memProtection: # When memory usage > memoryHighWaterLevel, all dml requests would be rejected; # When memoryLowWaterLevel < memory usage < memoryHighWaterLevel, reduce the dml rate; diff --git a/internal/rootcoord/quota_center.go b/internal/rootcoord/quota_center.go index 41ecfd6aea..39c0c0a13d 100644 --- a/internal/rootcoord/quota_center.go +++ b/internal/rootcoord/quota_center.go @@ -920,9 +920,11 @@ func (q *QuotaCenter) calculateWriteRates() error { internalpb.RateType_DMLUpsert, collectionLimiter) q.guaranteeMinRate(getCollectionRateLimitConfig(collectionProps, common.CollectionDeleteRateMinKey), internalpb.RateType_DMLDelete, collectionLimiter) - log.RatedDebug(10, "QuotaCenter cool write rates off done", - zap.Int64("collectionID", collection), - zap.Float64("factor", factor)) + if factor < 1.0 { + log.RatedDebug(10, "QuotaCenter cool write rates off done", + zap.Int64("collectionID", collection), + zap.Float64("factor", factor)) + } } if len(ttCollections) > 0 { @@ -1014,16 +1016,17 @@ func (q *QuotaCenter) getTimeTickDelayFactor(ts Timestamp) map[int64]float64 { continue } factor := float64(maxDelay.Nanoseconds()-curMaxDelay.Nanoseconds()) / float64(maxDelay.Nanoseconds()) - if factor <= 0.9 { + if factor <= 0.95 { log.RatedWarn(10, "QuotaCenter: limit writing due to long timeTick delay", zap.Int64("collectionID", collectionID), zap.Time("curTs", t1), zap.Duration("delay", curMaxDelay), zap.Duration("MaxDelay", maxDelay), zap.Float64("factor", factor)) + collectionFactor[collectionID] = factor + continue } - - collectionFactor[collectionID] = factor + collectionFactor[collectionID] = 1.0 } return collectionFactor diff --git a/pkg/util/paramtable/quota_param.go b/pkg/util/paramtable/quota_param.go index 9456382a1e..05a5f69e95 100644 --- a/pkg/util/paramtable/quota_param.go +++ b/pkg/util/paramtable/quota_param.go @@ -1689,7 +1689,7 @@ specific conditions, such as memory of nodes to water marker), ` + "true" + ` me } p.TtProtectionEnabled.Init(base.mgr) - const defaultMaxTtDelay = "300.0" + const defaultMaxTtDelay = "1200.0" p.MaxTimeTickDelay = ParamItem{ Key: "quotaAndLimits.limitWriting.ttProtection.maxTimeTickDelay", Version: "2.2.0", diff --git a/pkg/util/paramtable/quota_param_test.go b/pkg/util/paramtable/quota_param_test.go index 0e555b777a..62dd77eb05 100644 --- a/pkg/util/paramtable/quota_param_test.go +++ b/pkg/util/paramtable/quota_param_test.go @@ -207,7 +207,7 @@ func TestQuotaParam(t *testing.T) { t.Run("test limit writing", func(t *testing.T) { assert.False(t, qc.ForceDenyWriting.GetAsBool()) assert.Equal(t, true, qc.TtProtectionEnabled.GetAsBool()) - assert.Equal(t, 300, qc.MaxTimeTickDelay.GetAsInt()) + assert.Equal(t, 1200, qc.MaxTimeTickDelay.GetAsInt()) assert.Equal(t, defaultLowWaterLevel, qc.DataNodeMemoryLowWaterLevel.GetAsFloat()) assert.Equal(t, defaultHighWaterLevel, qc.DataNodeMemoryHighWaterLevel.GetAsFloat()) assert.Equal(t, defaultLowWaterLevel, qc.QueryNodeMemoryLowWaterLevel.GetAsFloat())