From b99164f43dfea996a6e2997e737ff48741d27c7e Mon Sep 17 00:00:00 2001
From: congqixia <congqi.xia@zilliz.com>
Date: Thu, 18 Nov 2021 22:29:40 +0800
Subject: [PATCH] Add datacoord GC configuration (#12061)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
---
 configs/milvus.yaml                     |  6 ++++++
 internal/datacoord/garbage_collector.go | 10 ++--------
 internal/datacoord/param_table.go       | 24 ++++++++++++++++++++++++
 internal/datacoord/server.go            |  6 +++---
 4 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
index 8f0e66d44a..69a5473857 100644
--- a/configs/milvus.yaml
+++ b/configs/milvus.yaml
@@ -179,6 +179,12 @@ dataCoord:
   compaction:
     retentionDuration: 432000 # 5 days in seconds
 
+  gc:
+    interval: 3600 # gc interval in seconds
+    missingTolerance: 86400 # file meta missing tolerance duration in seconds, 60*24
+    dropTolerance: 86400 # file belongs to dropped entity tolerance duration in seconds, 60*24
+
+
 dataNode:
   port: 21124
 
diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go
index 7eae7e6028..e80db26886 100644
--- a/internal/datacoord/garbage_collector.go
+++ b/internal/datacoord/garbage_collector.go
@@ -26,14 +26,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const (
-	// temp solution use const
-	// maybe put garbage config into config files in the future
-	defaultGcInterval       = 1 * time.Hour
-	defaultMissingTolerance = 24 * time.Hour // 1day
-	defaultDropTolerance    = 24 * time.Hour // 1day
-)
-
 // GcOption garbage collection options
 type GcOption struct {
 	cli              *minio.Client // OSS client
@@ -59,6 +51,8 @@ type garbageCollector struct {
 
 // newGarbageCollector create garbage collector with meta and option
 func newGarbageCollector(meta *meta, opt GcOption) *garbageCollector {
+	log.Info("GC with option", zap.Bool("enabled", opt.enabled), zap.Duration("interval", opt.checkInterval),
+		zap.Duration("missingTolerance", opt.missingTolerance), zap.Duration("dropTolerance", opt.dropTolerance))
 	return &garbageCollector{
 		meta:    meta,
 		option:  opt,
diff --git a/internal/datacoord/param_table.go b/internal/datacoord/param_table.go
index a6dd7b922f..329f323f05 100644
--- a/internal/datacoord/param_table.go
+++ b/internal/datacoord/param_table.go
@@ -81,6 +81,11 @@ type ParamTable struct {
 	EnableGarbageCollection bool
 
 	CompactionRetentionDuration int64
+
+	// Garbage Collection
+	GCInterval         time.Duration
+	GCMissingTolerance time.Duration
+	GCDropTolerance    time.Duration
 }
 
 // Params is a package scoped variable of type ParamTable.
@@ -130,6 +135,11 @@ func (p *ParamTable) Init() {
 	p.initMinioRootPath()
 
 	p.initCompactionRetentionDuration()
+
+	p.initEnableGarbageCollection()
+	p.initGCInterval()
+	p.initGCMissingTolerance()
+	p.initGCDropTolerance()
 }
 
 // InitOnce ensures param table is a singleton
@@ -298,10 +308,24 @@ func (p *ParamTable) initEnableCompaction() {
 	p.EnableCompaction = p.ParseBool("dataCoord.enableCompaction", false)
 }
 
+// -- GC --
+
 func (p *ParamTable) initEnableGarbageCollection() {
 	p.EnableGarbageCollection = p.ParseBool("dataCoord.enableGarbageCollection", false)
 }
 
+func (p *ParamTable) initGCInterval() {
+	p.GCInterval = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.interval", 60*60)) * time.Second
+}
+
+func (p *ParamTable) initGCMissingTolerance() {
+	p.GCMissingTolerance = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.missingTolerance", 24*60*60)) * time.Second
+}
+
+func (p *ParamTable) initGCDropTolerance() {
+	p.GCDropTolerance = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.dropTolerance", 24*60*60)) * time.Second
+}
+
 // --- MinIO ---
 func (p *ParamTable) initMinioAddress() {
 	endpoint, err := p.Load("_MinioAddress")
diff --git a/internal/datacoord/server.go b/internal/datacoord/server.go
index 57bff965f9..a6d9d2cec3 100644
--- a/internal/datacoord/server.go
+++ b/internal/datacoord/server.go
@@ -336,9 +336,9 @@ func (s *Server) initGarbageCollection() error {
 		bucketName: Params.MinioBucketName,
 		rootPath:   Params.MinioRootPath,
 
-		checkInterval:    defaultGcInterval,
-		missingTolerance: defaultMissingTolerance,
-		dropTolerance:    defaultMissingTolerance,
+		checkInterval:    Params.GCInterval,
+		missingTolerance: Params.GCMissingTolerance,
+		dropTolerance:    Params.GCDropTolerance,
 	})
 	return nil
 }