From dd682658dab99c7aaa352485be0a96a7aeca2f16 Mon Sep 17 00:00:00 2001
From: congqixia <congqi.xia@zilliz.com>
Date: Tue, 28 Nov 2023 07:00:28 +0800
Subject: [PATCH] enhance: Add channel checkpoint ts lag metric (#28767)

See also #28765

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
---
 internal/datacoord/meta.go       |  3 +++
 pkg/metrics/datacoord_metrics.go | 12 ++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go
index b5e014a5ea..74d66c5dbd 100644
--- a/internal/datacoord/meta.go
+++ b/internal/datacoord/meta.go
@@ -42,6 +42,7 @@ import (
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/metautil"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/timerecord"
 	"github.com/milvus-io/milvus/pkg/util/tsoutil"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
@@ -1268,6 +1269,8 @@ func (m *meta) UpdateChannelCheckpoint(vChannel string, pos *msgpb.MsgPosition)
 			zap.Uint64("ts", pos.GetTimestamp()),
 			zap.ByteString("msgID", pos.GetMsgID()),
 			zap.Time("time", ts))
+		metrics.DataCoordCheckpointLag.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), vChannel).
+			Set(float64(time.Since(ts).Milliseconds()))
 	}
 	return nil
 }
diff --git a/pkg/metrics/datacoord_metrics.go b/pkg/metrics/datacoord_metrics.go
index 32300e4a3b..f36f51e707 100644
--- a/pkg/metrics/datacoord_metrics.go
+++ b/pkg/metrics/datacoord_metrics.go
@@ -117,6 +117,17 @@ var (
 			channelNameLabelName,
 		})
 
+	DataCoordCheckpointLag = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.DataCoordRole,
+			Name:      "channel_checkpoint_ts_lag_ms",
+			Help:      "channel checkpoint timestamp lag in milliseconds",
+		}, []string{
+			nodeIDLabelName,
+			channelNameLabelName,
+		})
+
 	DataCoordStoredBinlogSize = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: milvusNamespace,
@@ -267,6 +278,7 @@ func RegisterDataCoord(registry *prometheus.Registry) {
 	registry.MustRegister(DataCoordNumStoredRows)
 	registry.MustRegister(DataCoordNumStoredRowsCounter)
 	registry.MustRegister(DataCoordConsumeDataNodeTimeTickLag)
+	registry.MustRegister(DataCoordCheckpointLag)
 	registry.MustRegister(DataCoordStoredBinlogSize)
 	registry.MustRegister(DataCoordSegmentBinLogFileCount)
 	registry.MustRegister(DataCoordDmlChannelNum)