admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: set 72h for wal retention (#42910)

Browse Source 
issue: #42706

Signed-off-by: chyezh <chyezh@outlook.com>
This commit is contained in:
Zhen Ye 2025-06-27 17:36:43 +08:00 committed by GitHub
parent cfe7fd356d
commit 8367e4ec6a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/milvus.yaml
View File

@ -1287,14 +1287,14 @@ streaming:
# Every time the checkpoint is persisted, the checkpoint will be sampled and used to be a candidate of truncate checkpoint. # Every time the checkpoint is persisted, the checkpoint will be sampled and used to be a candidate of truncate checkpoint.
# More samples, more frequent truncate, more memory usage. # More samples, more frequent truncate, more memory usage.
sampleInterval: 30m sampleInterval: 30m
# The retention interval of wal truncate, 26h by default. # The retention interval of wal truncate, 72h by default.
# If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint. # If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint.
# Greater the interval, more wal storage usage, more redundant data in wal. # Greater the interval, more wal storage usage, more redundant data in wal.
# Because current query path doesn't promise the read operation not happen before the truncate point, # Because current query path doesn't promise the read operation not happen before the truncate point,
# retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path. # retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path.
# If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost. # If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost.
# because the wal truncate operation is implemented by pulsar consumer. # because the wal truncate operation is implemented by pulsar consumer.
retentionInterval: 26h retentionInterval: 72h
# Any configuration related to the knowhere vector search engine # Any configuration related to the knowhere vector search engine
knowhere: knowhere:

2
internal/streamingnode/server/wal/recovery/config_test.go
View File

@ -56,5 +56,5 @@ func TestTruncatorConfig(t *testing.T) {
cfg := newTruncatorConfig() cfg := newTruncatorConfig()
assert.Equal(t, 30*time.Minute, cfg.sampleInterval) assert.Equal(t, 30*time.Minute, cfg.sampleInterval)
assert.Equal(t, 26*time.Hour, cfg.retentionInterval) assert.Equal(t, 72*time.Hour, cfg.retentionInterval)
} }

4
pkg/util/paramtable/component_param.go
View File

@ -5910,14 +5910,14 @@ More samples, more frequent truncate, more memory usage.`,
p.WALTruncateRetentionInterval = ParamItem{ p.WALTruncateRetentionInterval = ParamItem{
Key: "streaming.walTruncate.retentionInterval", Key: "streaming.walTruncate.retentionInterval",
Version: "2.6.0", Version: "2.6.0",
Doc: `The retention interval of wal truncate, 26h by default. Doc: `The retention interval of wal truncate, 72h by default.
If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint. If the sampled checkpoint is older than this interval, it will be used to truncate wal checkpoint.
Greater the interval, more wal storage usage, more redundant data in wal. Greater the interval, more wal storage usage, more redundant data in wal.
Because current query path doesn't promise the read operation not happen before the truncate point, Because current query path doesn't promise the read operation not happen before the truncate point,
retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path. retention interval should be greater than the dataCoord.segment.maxLife to avoid the message lost at query path.
If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost. If the wal is pulsar, the pulsar should close the subscription expiration to avoid the message lost.
because the wal truncate operation is implemented by pulsar consumer.`, because the wal truncate operation is implemented by pulsar consumer.`,
DefaultValue: "26h", DefaultValue: "72h",
Export: true, Export: true,
} }
p.WALTruncateRetentionInterval.Init(base.mgr) p.WALTruncateRetentionInterval.Init(base.mgr)

2
pkg/util/paramtable/component_param_test.go
View File

@ -654,7 +654,7 @@ func TestComponentParam(t *testing.T) {
assert.Equal(t, float64(0.2), params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.GetAsFloat()) assert.Equal(t, float64(0.2), params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.GetAsFloat())
assert.Equal(t, float64(0.1), params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.GetAsFloat()) assert.Equal(t, float64(0.1), params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.GetAsFloat())
assert.Equal(t, 30*time.Minute, params.StreamingCfg.WALTruncateSampleInterval.GetAsDurationByParse()) assert.Equal(t, 30*time.Minute, params.StreamingCfg.WALTruncateSampleInterval.GetAsDurationByParse())
assert.Equal(t, 26*time.Hour, params.StreamingCfg.WALTruncateRetentionInterval.GetAsDurationByParse()) assert.Equal(t, 72*time.Hour, params.StreamingCfg.WALTruncateRetentionInterval.GetAsDurationByParse())
params.Save(params.StreamingCfg.WALBalancerTriggerInterval.Key, "50s") params.Save(params.StreamingCfg.WALBalancerTriggerInterval.Key, "50s")
params.Save(params.StreamingCfg.WALBalancerBackoffInitialInterval.Key, "50s") params.Save(params.StreamingCfg.WALBalancerBackoffInitialInterval.Key, "50s")