From 8367e4ec6a45194d8c77da2c5444b6104d82d91a Mon Sep 17 00:00:00 2001 From: Zhen Ye Date: Fri, 27 Jun 2025 17:36:43 +0800 Subject: [PATCH] fix: set 72h for wal retention (#42910) issue: #42706 Signed-off-by: chyezh --- configs/milvus.yaml | 4 ++-- internal/streamingnode/server/wal/recovery/config_test.go | 2 +- pkg/util/paramtable/component_param.go | 4 ++-- pkg/util/paramtable/component_param_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 22e942710b..a19159f3d1 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -1287,14 +1287,14 @@ streaming: # Every time the checkpoint is persisted, the checkpoint will be sampled and used to be a candidate of truncate checkpoint. # More samples, more frequent truncate, more memory usage. sampleInterval: 30m - # The retention interval of wal truncate, 26h by default. + # The retention interval of wal truncate, 72h by default. # If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint. # Greater the interval, more wal storage usage, more redundant data in wal. # Because current query path doesn't promise the read operation not happen before the truncate point, # retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path. # If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost. # because the wal truncate operation is implemented by pulsar consumer. - retentionInterval: 26h + retentionInterval: 72h # Any configuration related to the knowhere vector search engine knowhere: diff --git a/internal/streamingnode/server/wal/recovery/config_test.go b/internal/streamingnode/server/wal/recovery/config_test.go index 238c79e4b9..779440277d 100644 --- a/internal/streamingnode/server/wal/recovery/config_test.go +++ b/internal/streamingnode/server/wal/recovery/config_test.go @@ -56,5 +56,5 @@ func TestTruncatorConfig(t *testing.T) { cfg := newTruncatorConfig() assert.Equal(t, 30*time.Minute, cfg.sampleInterval) - assert.Equal(t, 26*time.Hour, cfg.retentionInterval) + assert.Equal(t, 72*time.Hour, cfg.retentionInterval) } diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 09031e1e7d..5b8f01a30c 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -5910,14 +5910,14 @@ More samples, more frequent truncate, more memory usage.`, p.WALTruncateRetentionInterval = ParamItem{ Key: "streaming.walTruncate.retentionInterval", Version: "2.6.0", - Doc: `The retention interval of wal truncate, 26h by default. + Doc: `The retention interval of wal truncate, 72h by default. If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint. Greater the interval, more wal storage usage, more redundant data in wal. Because current query path doesn't promise the read operation not happen before the truncate point, retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path. If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost. because the wal truncate operation is implemented by pulsar consumer.`, - DefaultValue: "26h", + DefaultValue: "72h", Export: true, } p.WALTruncateRetentionInterval.Init(base.mgr) diff --git a/pkg/util/paramtable/component_param_test.go b/pkg/util/paramtable/component_param_test.go index 5445caa442..cac71cfd51 100644 --- a/pkg/util/paramtable/component_param_test.go +++ b/pkg/util/paramtable/component_param_test.go @@ -654,7 +654,7 @@ func TestComponentParam(t *testing.T) { assert.Equal(t, float64(0.2), params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.GetAsFloat()) assert.Equal(t, float64(0.1), params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.GetAsFloat()) assert.Equal(t, 30*time.Minute, params.StreamingCfg.WALTruncateSampleInterval.GetAsDurationByParse()) - assert.Equal(t, 26*time.Hour, params.StreamingCfg.WALTruncateRetentionInterval.GetAsDurationByParse()) + assert.Equal(t, 72*time.Hour, params.StreamingCfg.WALTruncateRetentionInterval.GetAsDurationByParse()) params.Save(params.StreamingCfg.WALBalancerTriggerInterval.Key, "50s") params.Save(params.StreamingCfg.WALBalancerBackoffInitialInterval.Key, "50s")