diff --git a/deployments/monitor/grafana/README.md b/deployments/monitor/grafana/README.md
index 21d58be482..8e36958c2e 100644
--- a/deployments/monitor/grafana/README.md
+++ b/deployments/monitor/grafana/README.md
@@ -24,7 +24,7 @@ We recommend reading [Milvus monitoring framework overview](monitor_overview.md)
| Mutation Send Latency | The average latency and the 99th percentile of the latency of sending insertion or deletion requests by each proxy within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_proxy_mutation_send_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_send_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id, msg_type) ``` | `milvus_proxy_mutation_send_latency` | The latency of sending insertion or deletion requests. |
| Cache Hit Rate | The average cache hit rate of operations including `GeCollectionID`, `GetCollectionInfo `, and `GetCollectionSchema` per second within the past two minutes. | ``` sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace", cache_state="hit"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])/120) by(cache_name, pod, node_id) ``` | `milvus_proxy_cache_hit_count` | The statistics of hit and failure rate of each cache reading operation. |
| Cache Update Latency | The average latency and the 99th percentile of cache update latency by proxy within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) ``` | `milvus_proxy_cache_update_latency` | The latency of updating cache each time. |
-| Sync Time | The average, maximum, and minimum number of epoch time synced by each proxy in its corresponding physical channel. | ``` avg(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) max(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) min(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_proxy_sync_epoch_time ` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970).
There is a default `ChannelName` apart from the physical channels. |
+| Sync Time | The average, maximum, and minimum number of epoch time synced by each proxy in its corresponding physical channel. | ``` avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_proxy_tt_lag_ms ` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970).
There is a default `ChannelName` apart from the physical channels. |
| Apply PK Latency | The average latency and the 99th percentile of primary key application latency by each proxy within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) ``` | `milvus_proxy_apply_pk_latency` | The latency of applying primary key. |
| Apply Timestamp Latency | The average latency and the 99th percentile of timestamp application latency by each proxy within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (pod, node_id) ``` | `milvus_proxy_apply_timestamp_latency` | The latency of applying timestamp. |
| DQL Request Rate | The status and number of DQL requests received per second by each proxy within the past two minutes.
DQL requests include `DescribeCollection`, `DescribeIndex`, `GetCollectionStatistics`, `HasCollection`, `Search`, `Query`, `ShowPartitions`, etc. This panel specifically shows the total number and the number of successful DQL requests. | ``` sum(increase(milvus_proxy_dql_req_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])/120) by(function_name, status, pod, node_id) ``` | `milvus_proxy_dql_req_count` | The number of all types of DQL requests. |
@@ -45,7 +45,7 @@ We recommend reading [Milvus monitoring framework overview](monitor_overview.md)
| Panel | Panel description | PromQL (Prometheus query language) | The Milvus metrics used | Milvus metrics description |
|---|---|---|---|---|
| Proxy Node Num | The number of proxies created. | ``` sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_rootcoord_proxy_num` | The number of proxies. |
-| Sync Time | The average, maximum, and minimum number of epoch time synced by each root coord in each physical channel (PChannel). | ``` avg(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) max(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) min(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_rootcoord_sync_epoch_time` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970). |
+| Sync Time | The average, maximum, and minimum number of epoch time synced by each root coord in each physical channel (PChannel). | ``` avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_rootcoord_produce_tt_lag_ms` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970). |
| DDL Request Rate | The status and number of DDL requests per second within the past two minutes. | ``` sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])/120) by (status, function_name) ``` | `milvus_rootcoord_ddl_req_count` | The total number of DDL requests including `CreateCollection`, `DescribeCollection`, `DescribeSegments`, `HasCollection`, `ShowCollections`, `ShowPartitions`, and `ShowSegments`. |
| DDL Request Latency | The average latency and the 99th percentile of DDL request latency within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by (function_name) ``` | `milvus_rootcoord_ddl_req_latency` | The latency of all types of DDL requests. |
| Sync Timetick Latency | The average latency and the 99th percentile of the time used by root coord to sync all timestamp to PChannel within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) ``` | `milvus_rootcoord_sync_timetick_latency` | the time used by root coord to sync all timestamp to pchannel. |
@@ -120,7 +120,7 @@ We recommend reading [Milvus monitoring framework overview](monitor_overview.md)
| Collection Num | The number of collections recorded in metadata by data coord. | ``` sum(milvus_datacoord_collection_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_datacoord_collection_num` | The number of collections recorded in metadata by data coord. |
| Stored Rows | The accumulated number of rows of valid and flushed data in data coord. | ``` sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_datacoord_stored_rows_num` | The accumulated number of rows of valid and flushed data in data coord. |
| Stored Rows Rate | The average number of rows flushed per second within the past two minutes. | ``` sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])/120) by (pod, node_id) ``` | `milvus_datacoord_stored_rows_count` | The accumulated number of rows flushed by data coord. |
-| Sync Time | The average, maximum, and minimum number of epoch time synced by data coord in each physical channel. | ``` avg(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) max(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_datacoord_sync_epoch_time` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970). |
+| Sync Time | The average, maximum, and minimum number of epoch time synced by data coord in each physical channel. | ``` avg(milvus_datacoord_consumer_datanode_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) max(milvus_datacoord_consumer_datanode_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) min(milvus_datacoord_consumer_datanode_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (app_kubernetes_io_instance) ``` | `milvus_datacoord_consumer_datanode_tt_lag_ms` | Each physical channel's epoch time (Unix time, the milliseconds passed ever since January 1, 1970). |
@@ -133,7 +133,7 @@ We recommend reading [Milvus monitoring framework overview](monitor_overview.md)
| Flush Data Size Rate | The size of each flushed message recorded per second by each data node within the past two minutes. | ``` sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])/120) by (msg_type, pod, node_id) ``` | `milvus_datanode_flushed_data_size` | The size of each flushed message.
Currently, streaming messages counted by data node only include insertion and deletion messages. |
| Consumer Num | The number of consumers created on each data node. | ``` sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_datanode_consumer_num` | The number of consumers created on each data node.
Each flowgraph corresponds to a consumer. |
| Producer Num | The number of producers created on each data node. | ``` sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_datanode_producer_num` | The number of consumers created on each data node.
Each shard in a collection corresponds to a delta channel producer and a timetick channel producer. |
-| Sync Time | The average, maximum, and minimum number of epoch time synced by each data node in all physical topics. | ``` avg(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) max(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) min(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_datanode_sync_epoch_time` | The epoch time (Unix time, the milliseconds passed ever since January 1, 1970.) of each physical topic on a data node. |
+| Sync Time | The average, maximum, and minimum number of epoch time synced by each data node in all physical topics. | ``` avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_datanode_produce_tt_lag_ms` | The epoch time (Unix time, the milliseconds passed ever since January 1, 1970.) of each physical topic on a data node. |
| Unflushed Segment Num | The number of unflushed segments created on each data node. | ``` sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}) by (pod, node_id) ``` | `milvus_datanode_unflushed_segment_num` | The number of unflushed segments created on each data node. |
| Encode Buffer Latency | The average latency and the 99th percentile of the time used to encode a buffer by each data node within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by(pod, node_id) ``` | `milvus_datanode_encode_buffer_latency` | The time each data node takes to encode a buffer. |
| Save Data Latency | The average latency and the 99th percentile of the time used to write a buffer into the storage layer by each data node within the past two minutes. | p99:
``` histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m]))) ```
avg:
``` sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~"$instance", app_kubernetes_io_name="$app_name", namespace="$namespace"}[2m])) by(pod, node_id) ``` | `milvus_datanode_save_latency` | The time each data node takes to write a buffer into the storage layer. |
diff --git a/deployments/monitor/grafana/milvus-dashboard.json b/deployments/monitor/grafana/milvus-dashboard.json
index 41bc88e0e0..9ff7b0ec54 100644
--- a/deployments/monitor/grafana/milvus-dashboard.json
+++ b/deployments/monitor/grafana/milvus-dashboard.json
@@ -1,35 +1,4 @@
{
- "__inputs": [
- {
- "name": "DS_PROMETHEUS",
- "label": "prometheus",
- "description": "",
- "type": "datasource",
- "pluginId": "prometheus",
- "pluginName": "Prometheus"
- }
- ],
- "__elements": [],
- "__requires": [
- {
- "type": "grafana",
- "id": "grafana",
- "name": "Grafana",
- "version": "8.3.3"
- },
- {
- "type": "panel",
- "id": "graph",
- "name": "Graph (old)",
- "version": ""
- },
- {
- "type": "datasource",
- "id": "prometheus",
- "name": "Prometheus",
- "version": "1.0.0"
- }
- ],
"annotations": {
"list": [
{
@@ -52,8 +21,8 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": null,
- "iteration": 1667373267980,
+ "id": 34,
+ "iteration": 1667533774069,
"links": [],
"liveNow": false,
"panels": [
@@ -1395,9 +1364,9 @@
"dashes": false,
"datasource": {
"type": "prometheus",
- "uid": "$datasource"
+ "uid": "P1809F7CD0C75ACF3"
},
- "description": "Average, maximum and minimum values of the timestamps for all physical channel.",
+ "description": "Average, maximum and minimum values of the timestamps for time tick behind.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1439,7 +1408,8 @@
"uid": "$datasource"
},
"exemplar": true,
- "expr": "avg(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{pod}}-{{node_id}}-avg",
@@ -1452,7 +1422,7 @@
"uid": "$datasource"
},
"exemplar": true,
- "expr": "max(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
"hide": false,
"interval": "",
"legendFormat": "{{pod}}-{{node_id}}-max",
@@ -1464,16 +1434,28 @@
"uid": "$datasource"
},
"exemplar": true,
- "expr": "min(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
"hide": false,
"interval": "",
"legendFormat": "{{pod}}-{{node_id}}-min",
"refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "exemplar": true,
+ "expr": "",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "D"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Sync Time",
+ "title": "Produced Timetick Lag Behind Now",
"tooltip": {
"shared": true,
"sort": 0,
@@ -1488,8 +1470,8 @@
"yaxes": [
{
"$$hashKey": "object:536",
- "decimals": 12,
- "format": "short",
+ "decimals": 0,
+ "format": "ms",
"logBase": 1,
"show": true
},
@@ -2178,7 +2160,7 @@
}
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@@ -2186,1350 +2168,1349 @@
"y": 43
},
"id": 123155,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of proxy nodes which has register with etcd",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 2
- },
- "hiddenSeries": false,
- "id": 123141,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Proxy Node Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": " Timestamps for each physical channel",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 2
- },
- "hiddenSeries": false,
- "id": 123335,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "avg(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "avg time tick",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "max(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "max time tick",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "min time tick",
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sync Time",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 12,
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of DDL request",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 2
- },
- "hiddenSeries": false,
- "id": 123347,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)",
- "interval": "",
- "legendFormat": "{{function_name}}-{{status}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DDL Request Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1456",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1457",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 8
- },
- "hiddenSeries": false,
- "id": 123337,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{function_name}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{function_name}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DDL Request Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 8
- },
- "hiddenSeries": false,
- "id": 123338,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-latency",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-latency",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sync Timetick Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of IDs assigned by RootCoord",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 8
- },
- "hiddenSeries": false,
- "id": 123221,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)",
- "interval": "",
- "legendFormat": "total",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "ID Alloc Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1456",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1457",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "RoootCoord current latest timestamp",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 14
- },
- "hiddenSeries": false,
- "id": 123339,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "tiestamp",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Timestamp",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "RootCoord stores pre-assigned timestamps in the metastore",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 14
- },
- "hiddenSeries": false,
- "id": 123340,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "tiestamp",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Timestamp Saved",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of collections.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 14
- },
- "hiddenSeries": false,
- "id": 123342,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Collection Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of partitions.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 123344,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Partition Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of DML channels",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 123345,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DML Channel Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of Msgstream objects.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 123346,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Msgstream Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of credential",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 26
- },
- "hiddenSeries": false,
- "id": 123343,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "num",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Credential Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The max time tick delay of flow graphs. unit ms",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 26
- },
- "hiddenSeries": false,
- "id": 123383,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{role_name}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Time Tick Delay",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:4353",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:4354",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
+ "panels": [],
"title": "Root Coordinator",
"type": "row"
},
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of proxy nodes which has register with etcd",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 123141,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Proxy Node Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick behind.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 123384,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Produced Timetick Lag Behind Now",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 123338,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-latency",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-latency",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Produced Timetick Time Taken",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "ms",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 50
+ },
+ "hiddenSeries": false,
+ "id": 123337,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{function_name}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{function_name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DDL Request Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "RootCoord stores pre-assigned timestamps in the metastore",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 50
+ },
+ "hiddenSeries": false,
+ "id": 123340,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "tiestamp",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Timestamp Saved",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of DDL request",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 50
+ },
+ "hiddenSeries": false,
+ "id": 123347,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)",
+ "interval": "",
+ "legendFormat": "{{function_name}}-{{status}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DDL Request Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1456",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1457",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "RoootCoord current latest timestamp",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 56
+ },
+ "hiddenSeries": false,
+ "id": 123339,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "tiestamp",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Timestamp",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of DML channels",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 56
+ },
+ "hiddenSeries": false,
+ "id": 123345,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DML Channel Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of IDs assigned by RootCoord",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 56
+ },
+ "hiddenSeries": false,
+ "id": 123221,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)",
+ "interval": "",
+ "legendFormat": "total",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "ID Alloc Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1456",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1457",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of partitions.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 62
+ },
+ "hiddenSeries": false,
+ "id": 123344,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Partition Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The max time tick delay of flow graphs. unit ms",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 62
+ },
+ "hiddenSeries": false,
+ "id": 123383,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{role_name}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Time Tick Delay",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of collections.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 62
+ },
+ "hiddenSeries": false,
+ "id": 123342,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Collection Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of credential",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 68
+ },
+ "hiddenSeries": false,
+ "id": 123343,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Credential Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of Msgstream objects.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 68
+ },
+ "hiddenSeries": false,
+ "id": 123346,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "num",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Msgstream Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:4353",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:4354",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 44
+ "y": 74
},
"id": 123246,
"panels": [
@@ -4489,4303 +4470,4938 @@
"type": "row"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 45
+ "y": 75
},
"id": 123244,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of the loaded collections in QueryNode.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 4
- },
- "hiddenSeries": false,
- "id": 123299,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Collection Loaded Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of the loaded partitions in QueryNode.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 4
- },
- "hiddenSeries": false,
- "id": 123303,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Partition Loaded Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of the loaded segment in QueryNode.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 4
- },
- "hiddenSeries": false,
- "id": 123305,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Segment Loaded Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of the queryable entities in QueryNode.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 123365,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Queryable Entity Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of dml virtual channels for QueryNode watch",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 123304,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "DML Virtual Channel",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of delta virtual channels for QueryNode watch",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 123306,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "hide": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Delta Virtual Channel",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of consumers in the QueryNode.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 16
- },
- "hiddenSeries": false,
- "id": 123307,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "hide": false,
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Consumer Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of searching requests.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 16
- },
- "hiddenSeries": false,
- "id": 123350,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Request Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:3414",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:3415",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 16
- },
- "hiddenSeries": false,
- "id": 123366,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Request Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 123372,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search in Queue Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of search segment over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 123374,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Segment Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 123310,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Segcore Request Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 123367,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Reduce Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of load segment over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 123311,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Load Segment Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of flowgraph",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 123312,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Flowgraph Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The length of the task queue for unsolved read requests",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 34
- },
- "hiddenSeries": false,
- "id": 123351,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Unsolved Read Task Length",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The length of the task queue of read requests to be executed",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 34
- },
- "hiddenSeries": false,
- "id": 123356,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Ready Read Task Length",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of read requests currently being executed in parallel",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 34
- },
- "hiddenSeries": false,
- "id": 123357,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Parallel Read Task Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "cpu utilization under scheduler evaluation",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 123358,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Estimate CPU Usage",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of original tasks contained in the merged search task",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 123352,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Group Size",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of queries for search requests",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 40
- },
- "hiddenSeries": false,
- "id": 123361,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search NQ",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of queries for the merged search requests",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 123360,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-milvus_querynode_search_group_nq",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Group NQ",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Top_K for search requests",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 123359,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Top_K",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Top_K for the merged search requests.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 46
- },
- "hiddenSeries": false,
- "id": 123362,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Search Group Top_K",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of evicted read requests.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 52
- },
- "hiddenSeries": false,
- "id": 123364,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Evicted Read Requests Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:3414",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:3415",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
+ "panels": [],
"title": "Query Node",
"type": "row"
},
{
- "collapsed": true,
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of the loaded collections in QueryNode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 76
+ },
+ "hiddenSeries": false,
+ "id": 123299,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Collection Loaded Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of the loaded partitions in QueryNode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 76
+ },
+ "hiddenSeries": false,
+ "id": 123303,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Partition Loaded Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of the loaded segment in QueryNode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 76
+ },
+ "hiddenSeries": false,
+ "id": 123305,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Segment Loaded Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick behind.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 82
+ },
+ "hiddenSeries": false,
+ "id": 123385,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TimeTick Lag Behind Now (Consumed Insert)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick behind.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 82
+ },
+ "hiddenSeries": false,
+ "id": 123386,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TimeTick Lag Behind Now (Consumed Delete)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of consuming message",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 82
+ },
+ "hiddenSeries": false,
+ "id": 123387,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-all",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Consumed Message Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "cps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of the queryable entities in QueryNode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 88
+ },
+ "hiddenSeries": false,
+ "id": 123365,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Queryable Entity Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of dml virtual channels for QueryNode watch",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 88
+ },
+ "hiddenSeries": false,
+ "id": 123304,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DML Virtual Channel",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of delta virtual channels for QueryNode watch",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 88
+ },
+ "hiddenSeries": false,
+ "id": 123306,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Delta Virtual Channel",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of consumers in the QueryNode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 94
+ },
+ "hiddenSeries": false,
+ "id": 123307,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Consumer Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of searching requests.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 94
+ },
+ "hiddenSeries": false,
+ "id": 123350,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Request Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 94
+ },
+ "hiddenSeries": false,
+ "id": 123366,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Request Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 100
+ },
+ "hiddenSeries": false,
+ "id": 123372,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search in Queue Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of search segment over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 100
+ },
+ "hiddenSeries": false,
+ "id": 123374,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Segment Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 100
+ },
+ "hiddenSeries": false,
+ "id": 123310,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Segcore Request Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 106
+ },
+ "hiddenSeries": false,
+ "id": 123367,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Reduce Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of load segment over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 106
+ },
+ "hiddenSeries": false,
+ "id": 123311,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Load Segment Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of flowgraph",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 106
+ },
+ "hiddenSeries": false,
+ "id": 123312,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Flowgraph Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The length of the task queue for unsolved read requests",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 112
+ },
+ "hiddenSeries": false,
+ "id": 123351,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Unsolved Read Task Length",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The length of the task queue of read requests to be executed",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 112
+ },
+ "hiddenSeries": false,
+ "id": 123356,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Ready Read Task Length",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of read requests currently being executed in parallel",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 112
+ },
+ "hiddenSeries": false,
+ "id": 123357,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Parallel Read Task Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "cpu utilization under scheduler evaluation",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 118
+ },
+ "hiddenSeries": false,
+ "id": 123358,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Estimate CPU Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of original tasks contained in the merged search task",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 118
+ },
+ "hiddenSeries": false,
+ "id": 123352,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Group Size",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of queries for search requests",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 118
+ },
+ "hiddenSeries": false,
+ "id": 123361,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search NQ",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of queries for the merged search requests",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 124
+ },
+ "hiddenSeries": false,
+ "id": 123360,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-milvus_querynode_search_group_nq",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Group NQ",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Top_K for search requests",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 124
+ },
+ "hiddenSeries": false,
+ "id": 123359,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Top_K",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Top_K for the merged search requests.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 124
+ },
+ "hiddenSeries": false,
+ "id": 123362,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Search Group Top_K",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of evicted read requests.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 130
+ },
+ "hiddenSeries": false,
+ "id": 123364,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Evicted Read Requests Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 46
+ "y": 136
},
"id": 123172,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of data nodes which has register with etcd.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 123207,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "total",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Data Node Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Number of segments with different states in the Meta of DataCoord",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 123267,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{segment_state}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Segment Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The number of rows of valid data accumulated in DataCoord that flushed.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 123269,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "row",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Stored Rows",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of data flushed.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 11
- },
- "hiddenSeries": false,
- "id": 123371,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Stored Rows Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:3414",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:3415",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "On the DataCoord, the average, max and min timestmp on each physical topic",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 11
- },
- "hiddenSeries": false,
- "id": 123270,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "avg(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "avg time tick",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "max(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "max time tick",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "min time tick",
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sync Time",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "decimals": 12,
- "format": "short",
- "label": "",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "binlog size of all collections/segments, unit byte",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 11
- },
- "hiddenSeries": false,
- "id": 123382,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "size",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Stored Binlog Size",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
+ "panels": [],
"title": "Data Coordinator",
"type": "row"
},
{
- "collapsed": true,
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of data nodes which has register with etcd.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 137
+ },
+ "hiddenSeries": false,
+ "id": 123207,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "total",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Data Node Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 137
+ },
+ "hiddenSeries": false,
+ "id": 123390,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TimeTick Lag Behind Now From DataNode",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The number of rows of valid data accumulated in DataCoord that flushed.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 137
+ },
+ "hiddenSeries": false,
+ "id": 123269,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "row",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Stored Rows",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of data flushed.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 143
+ },
+ "hiddenSeries": false,
+ "id": 123371,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Stored Rows Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Number of segments with different states in the Meta of DataCoord",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 143
+ },
+ "hiddenSeries": false,
+ "id": 123267,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{segment_state}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Segment Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "binlog size of all collections/segments, unit byte",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 143
+ },
+ "hiddenSeries": false,
+ "id": 123382,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "size",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Stored Binlog Size",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 47
+ "y": 149
},
"id": 123242,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of flowgraph",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 6
- },
- "hiddenSeries": false,
- "id": 123272,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Flowgraph Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of messages consumed for insert and delete operation.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 6
- },
- "hiddenSeries": false,
- "id": 123274,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Msg Rows Consum Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of each message that has been flushed.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 6
- },
- "hiddenSeries": false,
- "id": 123275,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Flush Data Size Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of consumers created on the DataNode",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 123276,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Consumer Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of prodecer created on the DataNode",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 123277,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Producer Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "On the DataNode, the average, max and min timestmp on each physical topic",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 123279,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "avg(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "max(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "max-{{pod}}-{{node_id}}",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "min(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "min-{{pod}}-{{node_id}}",
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Sync Time",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "Total number of segment that has been not flushed.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 18
- },
- "hiddenSeries": false,
- "id": 123280,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Unflushed Segment Num",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 18
- },
- "hiddenSeries": false,
- "id": 123282,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Encode Buffer Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 18
- },
- "hiddenSeries": false,
- "id": 123283,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Save Data Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of flush operete.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 24
- },
- "hiddenSeries": false,
- "id": 123284,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Flush Operate Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of auto flush operate.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 8,
- "y": 24
- },
- "hiddenSeries": false,
- "id": 123285,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Autoflush Operate Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "per-second increasing rate of flush requests.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 16,
- "y": 24
- },
- "hiddenSeries": false,
- "id": 123286,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
- "interval": "",
- "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
- "queryType": "randomWalk",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Flush Request Rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:101",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:102",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "description": "The 99th percentile and average latency of compaction over the last 2 minutes.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 6,
- "w": 8,
- "x": 0,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 123314,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
- "interval": "",
- "legendFormat": "p99-{{pod}}-{{node_id}}",
- "queryType": "randomWalk",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$datasource"
- },
- "exemplar": true,
- "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
- "hide": false,
- "interval": "",
- "legendFormat": "avg-{{pod}}-{{node_id}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Compaction Latency",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:161",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:162",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
+ "panels": [],
"title": "Data Node",
"type": "row"
},
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of flowgraph",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 150
+ },
+ "hiddenSeries": false,
+ "id": 123272,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Flowgraph Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of prodecer created on the DataNode",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 150
+ },
+ "hiddenSeries": false,
+ "id": 123277,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Producer Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 150
+ },
+ "hiddenSeries": false,
+ "id": 123393,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TimeTick Lag Behind Now (Consumed All)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P1809F7CD0C75ACF3"
+ },
+ "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 156
+ },
+ "hiddenSeries": false,
+ "id": 123388,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}-avg",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-max",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-min",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Produced TimeTick Lag Behind Now For DC",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "decimals": 0,
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of consuming message",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 156
+ },
+ "hiddenSeries": false,
+ "id": 123389,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Consumed Message Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "cps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of consumers created on the DataNode",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 162
+ },
+ "hiddenSeries": false,
+ "id": 123276,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Consumer Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 162
+ },
+ "hiddenSeries": false,
+ "id": 123282,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Encode Buffer Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of consuming message",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 162
+ },
+ "hiddenSeries": false,
+ "id": 123391,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Consumed Message Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:3414",
+ "format": "cps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:3415",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "Total number of segment that has been not flushed.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 168
+ },
+ "hiddenSeries": false,
+ "id": 123280,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Unflushed Segment Num",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of auto flush operate.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 8,
+ "y": 168
+ },
+ "hiddenSeries": false,
+ "id": 123285,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Autoflush Operate Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of messages consumed for insert and delete operation.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 168
+ },
+ "hiddenSeries": false,
+ "id": 123274,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Msg Rows Consumed Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "cps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of flush operete.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 174
+ },
+ "hiddenSeries": false,
+ "id": 123284,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Flush Operate Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of each message that has been flushed.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 174
+ },
+ "hiddenSeries": false,
+ "id": 123275,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Flush Data Size Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of compaction over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 0,
+ "y": 180
+ },
+ "hiddenSeries": false,
+ "id": 123314,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Compaction Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 180
+ },
+ "hiddenSeries": false,
+ "id": 123283,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
+ "interval": "",
+ "legendFormat": "p99-{{pod}}-{{node_id}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "avg-{{pod}}-{{node_id}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Save Data Latency",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:161",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:162",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "per-second increasing rate of flush requests.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 8,
+ "x": 16,
+ "y": 186
+ },
+ "hiddenSeries": false,
+ "id": 123286,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
+ "interval": "",
+ "legendFormat": "{{pod}}-{{node_id}}-{{status}}",
+ "queryType": "randomWalk",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Flush Request Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:101",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:102",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 48
+ "y": 192
},
"id": 123223,
"panels": [
@@ -9075,7 +9691,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 49
+ "y": 193
},
"id": 123231,
"panels": [
@@ -9693,7 +10309,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 50
+ "y": 194
},
"id": 123157,
"panels": [
@@ -9713,7 +10329,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 9
+ "y": 183
},
"hiddenSeries": false,
"id": 123202,
@@ -9803,7 +10419,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 9
+ "y": 183
},
"hiddenSeries": false,
"id": 123204,
@@ -9858,12 +10474,14 @@
},
"yaxes": [
{
+ "$$hashKey": "object:1340",
"format": "bytes",
"logBase": 1,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:1341",
"format": "bytes",
"logBase": 1,
"show": false
@@ -9888,7 +10506,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 9
+ "y": 183
},
"hiddenSeries": false,
"id": 123206,
@@ -9943,6 +10561,7 @@
},
"yaxes": [
{
+ "$$hashKey": "object:1281",
"format": "short",
"label": "",
"logBase": 1,
@@ -9950,6 +10569,7 @@
"show": true
},
{
+ "$$hashKey": "object:1282",
"format": "short",
"logBase": 1,
"show": true
@@ -9974,7 +10594,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 15
+ "y": 189
},
"hiddenSeries": false,
"id": 123135,
@@ -10059,7 +10679,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 15
+ "y": 189
},
"hiddenSeries": false,
"id": 123133,
@@ -10145,7 +10765,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 15
+ "y": 189
},
"hiddenSeries": false,
"id": 123125,
@@ -10229,7 +10849,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 21
+ "y": 195
},
"hiddenSeries": false,
"id": 123137,
@@ -10314,7 +10934,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 21
+ "y": 195
},
"hiddenSeries": false,
"id": 123131,
@@ -10397,7 +11017,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 21
+ "y": 195
},
"hiddenSeries": false,
"id": 123127,
@@ -10482,7 +11102,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 27
+ "y": 201
},
"hiddenSeries": false,
"id": 123129,
@@ -10567,7 +11187,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 27
+ "y": 201
},
"hiddenSeries": false,
"id": 123211,
@@ -10652,7 +11272,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 27
+ "y": 201
},
"hiddenSeries": false,
"id": 123212,
@@ -10737,7 +11357,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 33
+ "y": 207
},
"hiddenSeries": false,
"id": 123213,
@@ -10822,7 +11442,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 33
+ "y": 207
},
"hiddenSeries": false,
"id": 123215,
@@ -10907,7 +11527,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 33
+ "y": 207
},
"hiddenSeries": false,
"id": 123216,
@@ -10993,7 +11613,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 39
+ "y": 213
},
"hiddenSeries": false,
"id": 123217,
@@ -11084,7 +11704,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 39
+ "y": 213
},
"hiddenSeries": false,
"id": 123218,
@@ -11169,7 +11789,7 @@
"h": 6,
"w": 8,
"x": 16,
- "y": 39
+ "y": 213
},
"hiddenSeries": false,
"id": 123219,
@@ -11255,7 +11875,7 @@
"h": 6,
"w": 8,
"x": 0,
- "y": 45
+ "y": 219
},
"hiddenSeries": false,
"id": 123375,
@@ -11345,7 +11965,7 @@
"h": 6,
"w": 8,
"x": 8,
- "y": 45
+ "y": 219
},
"hiddenSeries": false,
"id": 123376,
@@ -11470,7 +12090,11 @@
"type": "textbox"
},
{
- "current": {},
+ "current": {
+ "selected": true,
+ "text": "milvus-ci",
+ "value": "milvus-ci"
+ },
"datasource": {
"uid": "$datasource"
},
@@ -11494,7 +12118,11 @@
"useTags": false
},
{
- "current": {},
+ "current": {
+ "selected": false,
+ "text": "md-20166-27-pr",
+ "value": "md-20166-27-pr"
+ },
"datasource": {
"uid": "$datasource"
},
@@ -11519,10 +12147,14 @@
},
{
"allValue": ".*",
- "current": {},
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
"datasource": {
"type": "prometheus",
- "uid": "${DS_PROMETHEUS}"
+ "uid": "prometheus"
},
"definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)",
"hide": 0,
@@ -11541,7 +12173,11 @@
"type": "query"
},
{
- "current": {},
+ "current": {
+ "selected": false,
+ "text": "md-20166-27-pr-milvus-datacoord-5bbc7b5f54-4265t",
+ "value": "md-20166-27-pr-milvus-datacoord-5bbc7b5f54-4265t"
+ },
"datasource": {
"uid": "$datasource"
},
@@ -11565,7 +12201,12 @@
"useTags": false
},
{
- "current": {},
+ "current": {
+ "isNone": true,
+ "selected": false,
+ "text": "None",
+ "value": ""
+ },
"datasource": {
"uid": "$datasource"
},
@@ -11591,7 +12232,7 @@
]
},
"time": {
- "from": "now-30m",
+ "from": "now-6h",
"to": "now"
},
"timepicker": {
@@ -11624,6 +12265,6 @@
"timezone": "browser",
"title": "Milvus2.0",
"uid": "uLf5cJ3Gz",
- "version": 23,
+ "version": 5,
"weekStart": ""
}
\ No newline at end of file
diff --git a/internal/datacoord/server.go b/internal/datacoord/server.go
index 997ebe801f..454cf67822 100644
--- a/internal/datacoord/server.go
+++ b/internal/datacoord/server.go
@@ -555,10 +555,11 @@ func (s *Server) handleTimetickMessage(ctx context.Context, ttMsg *msgstream.Dat
log.RatedWarn(60.0, "time tick lag behind for more than 1 minutes", zap.String("channel", ch), zap.Time("timetick", physical))
}
- utcT, _ := tsoutil.ParseHybridTs(ts)
-
+ sub := tsoutil.SubByNow(ts)
pChannelName := funcutil.ToPhysicalChannel(ch)
- metrics.DataCoordSyncEpoch.WithLabelValues(pChannelName).Set(float64(utcT))
+ metrics.DataCoordConsumeDataNodeTimeTickLag.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), pChannelName).
+ Set(float64(sub))
s.updateSegmentStatistics(ttMsg.GetSegmentsStats())
diff --git a/internal/datanode/flow_graph_dd_node.go b/internal/datanode/flow_graph_dd_node.go
index 5f4793b8bb..d0a5ee6b37 100644
--- a/internal/datanode/flow_graph_dd_node.go
+++ b/internal/datanode/flow_graph_dd_node.go
@@ -20,9 +20,10 @@ import (
"context"
"fmt"
"reflect"
- "strconv"
"sync/atomic"
+ "github.com/milvus-io/milvus/internal/util/timerecord"
+
"github.com/golang/protobuf/proto"
"github.com/opentracing/opentracing-go"
"go.uber.org/zap"
@@ -141,6 +142,9 @@ func (ddn *ddNode) Operate(in []Msg) []Msg {
log.Info("Stop compaction of vChannel", zap.String("vChannelName", ddn.vChannelName))
ddn.compactionExecutor.stopExecutingtaskByVChannelName(ddn.vChannelName)
fgMsg.dropCollection = true
+
+ pChan := funcutil.ToPhysicalChannel(ddn.vChannelName)
+ metrics.CleanupDataNodeCollectionMetrics(paramtable.GetNodeID(), ddn.collectionID, pChan)
}
case commonpb.MsgType_DropPartition:
@@ -172,7 +176,14 @@ func (ddn *ddNode) Operate(in []Msg) []Msg {
}
rateCol.Add(metricsinfo.InsertConsumeThroughput, float64(proto.Size(&imsg.InsertRequest)))
- metrics.DataNodeConsumeCounter.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.InsertLabel).Add(float64(proto.Size(&imsg.InsertRequest)))
+
+ metrics.DataNodeConsumeBytesCount.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.InsertLabel).
+ Add(float64(proto.Size(&imsg.InsertRequest)))
+
+ metrics.DataNodeConsumeMsgCount.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.InsertLabel, fmt.Sprint(ddn.collectionID)).
+ Inc()
log.Debug("DDNode receive insert messages",
zap.Int("numRows", len(imsg.GetRowIDs())),
@@ -195,7 +206,14 @@ func (ddn *ddNode) Operate(in []Msg) []Msg {
continue
}
rateCol.Add(metricsinfo.DeleteConsumeThroughput, float64(proto.Size(&dmsg.DeleteRequest)))
- metrics.DataNodeConsumeCounter.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.DeleteLabel).Add(float64(proto.Size(&dmsg.DeleteRequest)))
+
+ metrics.DataNodeConsumeBytesCount.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.DeleteLabel).
+ Add(float64(proto.Size(&dmsg.DeleteRequest)))
+
+ metrics.DataNodeConsumeMsgCount.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.DeleteLabel, fmt.Sprint(ddn.collectionID)).
+ Inc()
fgMsg.deleteMessages = append(fgMsg.deleteMessages, dmsg)
}
}
@@ -262,6 +280,8 @@ func (ddn *ddNode) isDropped(segID UniqueID) bool {
}
func (ddn *ddNode) forwardDeleteMsg(msgs []msgstream.TsMsg, minTs Timestamp, maxTs Timestamp) error {
+ tr := timerecord.NewTimeRecorder("forwardDeleteMsg")
+
if len(msgs) != 0 {
var msgPack = msgstream.MsgPack{
Msgs: msgs,
@@ -275,6 +295,10 @@ func (ddn *ddNode) forwardDeleteMsg(msgs []msgstream.TsMsg, minTs Timestamp, max
if err := ddn.sendDeltaTimeTick(maxTs); err != nil {
return err
}
+
+ metrics.DataNodeForwardDeleteMsgTimeTaken.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).
+ Observe(float64(tr.ElapseSpan().Milliseconds()))
return nil
}
diff --git a/internal/datanode/flow_graph_dmstream_input_node.go b/internal/datanode/flow_graph_dmstream_input_node.go
index c7107bdbfa..f4bc61ee89 100644
--- a/internal/datanode/flow_graph_dmstream_input_node.go
+++ b/internal/datanode/flow_graph_dmstream_input_node.go
@@ -31,6 +31,7 @@ import (
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/tsoutil"
+ "github.com/milvus-io/milvus/internal/util/typeutil"
)
// DmInputNode receives messages from message streams, packs messages between two timeticks, and passes all
@@ -79,6 +80,7 @@ func newDmInputNode(ctx context.Context, seekPos *internalpb.MsgPosition, dmNode
log.Info("datanode AsConsumer", zap.String("physical channel", pchannelName), zap.String("subName", consumeSubName), zap.Int64("collection ID", dmNodeConfig.collectionID))
name := fmt.Sprintf("dmInputNode-data-%d-%s", dmNodeConfig.collectionID, dmNodeConfig.vChannelName)
- node := flowgraph.NewInputNode(insertStream, name, dmNodeConfig.maxQueueLength, dmNodeConfig.maxParallelism)
+ node := flowgraph.NewInputNode(insertStream, name, dmNodeConfig.maxQueueLength, dmNodeConfig.maxParallelism,
+ typeutil.DataNodeRole, paramtable.GetNodeID(), dmNodeConfig.collectionID, metrics.AllLabel)
return node, nil
}
diff --git a/internal/datanode/flow_graph_insert_buffer_node.go b/internal/datanode/flow_graph_insert_buffer_node.go
index dcfb3dc7d9..9aff6392d7 100644
--- a/internal/datanode/flow_graph_insert_buffer_node.go
+++ b/internal/datanode/flow_graph_insert_buffer_node.go
@@ -648,9 +648,11 @@ func newInsertBufferNode(ctx context.Context, collID UniqueID, flushCh <-chan fl
},
}
msgPack.Msgs = append(msgPack.Msgs, &timeTickMsg)
- pt, _ := tsoutil.ParseHybridTs(ts)
+ sub := tsoutil.SubByNow(ts)
pChan := funcutil.ToPhysicalChannel(config.vChannelName)
- metrics.DataNodeTimeSync.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), pChan).Set(float64(pt))
+ metrics.DataNodeProduceTimeTickLag.
+ WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(collID), pChan).
+ Set(float64(sub))
return wTtMsgStream.Produce(&msgPack)
})
diff --git a/internal/metrics/datacoord_metrics.go b/internal/metrics/datacoord_metrics.go
index 5c46ea4bd6..c30508b36c 100644
--- a/internal/metrics/datacoord_metrics.go
+++ b/internal/metrics/datacoord_metrics.go
@@ -76,13 +76,16 @@ var (
Help: "count of all stored rows ever",
}, []string{})
- DataCoordSyncEpoch = prometheus.NewGaugeVec(
+ DataCoordConsumeDataNodeTimeTickLag = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.DataCoordRole,
- Name: "sync_epoch_time",
- Help: "synchronized unix epoch per physical channel",
- }, []string{channelNameLabelName})
+ Name: "consume_datanode_tt_lag_ms",
+ Help: "now time minus tt per physical channel",
+ }, []string{
+ nodeIDLabelName,
+ channelNameLabelName,
+ })
DataCoordStoredBinlogSize = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
@@ -146,6 +149,6 @@ func RegisterDataCoord(registry *prometheus.Registry) {
registry.MustRegister(DataCoordNumCollections)
registry.MustRegister(DataCoordNumStoredRows)
registry.MustRegister(DataCoordNumStoredRowsCounter)
- registry.MustRegister(DataCoordSyncEpoch)
+ registry.MustRegister(DataCoordConsumeDataNodeTimeTickLag)
registry.MustRegister(DataCoordStoredBinlogSize)
}
diff --git a/internal/metrics/datanode_metrics.go b/internal/metrics/datanode_metrics.go
index baef577e7a..f0013952cb 100644
--- a/internal/metrics/datanode_metrics.go
+++ b/internal/metrics/datanode_metrics.go
@@ -17,6 +17,8 @@
package metrics
import (
+ "fmt"
+
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/prometheus/client_golang/prometheus"
)
@@ -74,17 +76,42 @@ var (
nodeIDLabelName,
})
- DataNodeTimeSync = prometheus.NewGaugeVec(
+ DataNodeConsumeTimeTickLag = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.DataNodeRole,
- Name: "sync_epoch_time",
- Help: "synchronized unix epoch per physical channel",
+ Name: "consume_tt_lag_ms",
+ Help: "now time minus tt per physical channel",
}, []string{
nodeIDLabelName,
+ msgTypeLabelName,
+ collectionIDLabelName,
+ })
+
+ DataNodeProduceTimeTickLag = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: milvusNamespace,
+ Subsystem: typeutil.DataNodeRole,
+ Name: "produce_tt_lag_ms",
+ Help: "now time minus tt pts per physical channel",
+ }, []string{
+ nodeIDLabelName,
+ collectionIDLabelName,
channelNameLabelName,
})
+ DataNodeConsumeMsgCount = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: milvusNamespace,
+ Subsystem: typeutil.DataNodeRole,
+ Name: "consume_msg_count",
+ Help: "count of consumed msg",
+ }, []string{
+ nodeIDLabelName,
+ msgTypeLabelName,
+ collectionIDLabelName,
+ })
+
DataNodeNumUnflushedSegments = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
@@ -163,14 +190,23 @@ var (
statusLabelName,
})
- // DataNodeConsumeCounter counts the bytes DataNode consumed from message storage.
- DataNodeConsumeCounter = prometheus.NewCounterVec(
+ // DataNodeConsumeBytesCount counts the bytes DataNode consumed from message storage.
+ DataNodeConsumeBytesCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.DataNodeRole,
Name: "consume_counter",
Help: "",
}, []string{nodeIDLabelName, msgTypeLabelName})
+
+ DataNodeForwardDeleteMsgTimeTaken = prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: milvusNamespace,
+ Subsystem: typeutil.DataNodeRole,
+ Name: "forward_delete_msg_time_taken_ms",
+ Help: "forward delete message time taken",
+ Buckets: buckets, // unit: ms
+ }, []string{nodeIDLabelName})
)
//RegisterDataNode registers DataNode metrics
@@ -180,7 +216,7 @@ func RegisterDataNode(registry *prometheus.Registry) {
registry.MustRegister(DataNodeFlushedSize)
registry.MustRegister(DataNodeNumConsumers)
registry.MustRegister(DataNodeNumProducers)
- registry.MustRegister(DataNodeTimeSync)
+ registry.MustRegister(DataNodeConsumeTimeTickLag)
registry.MustRegister(DataNodeNumUnflushedSegments)
registry.MustRegister(DataNodeEncodeBufferLatency)
registry.MustRegister(DataNodeSave2StorageLatency)
@@ -188,5 +224,36 @@ func RegisterDataNode(registry *prometheus.Registry) {
registry.MustRegister(DataNodeAutoFlushBufferCount)
registry.MustRegister(DataNodeCompactionLatency)
registry.MustRegister(DataNodeFlushReqCounter)
- registry.MustRegister(DataNodeConsumeCounter)
+ registry.MustRegister(DataNodeConsumeMsgCount)
+ registry.MustRegister(DataNodeProduceTimeTickLag)
+ registry.MustRegister(DataNodeConsumeBytesCount)
+ registry.MustRegister(DataNodeForwardDeleteMsgTimeTaken)
+}
+
+func CleanupDataNodeCollectionMetrics(nodeID int64, collectionID int64, channel string) {
+ DataNodeConsumeTimeTickLag.
+ Delete(
+ prometheus.Labels{
+ nodeIDLabelName: fmt.Sprint(nodeID),
+ msgTypeLabelName: AllLabel,
+ collectionIDLabelName: fmt.Sprint(collectionID),
+ })
+
+ DataNodeProduceTimeTickLag.
+ Delete(
+ prometheus.Labels{
+ nodeIDLabelName: fmt.Sprint(nodeID),
+ collectionIDLabelName: fmt.Sprint(collectionID),
+ channelNameLabelName: channel,
+ })
+
+ for _, label := range []string{AllLabel, DeleteLabel, InsertLabel} {
+ DataNodeConsumeMsgCount.
+ Delete(
+ prometheus.Labels{
+ nodeIDLabelName: fmt.Sprint(nodeID),
+ msgTypeLabelName: label,
+ collectionIDLabelName: fmt.Sprint(collectionID),
+ })
+ }
}
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index e7d6cc800f..168ef0dbf2 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -34,13 +34,14 @@ const (
FailLabel = "fail"
TotalLabel = "total"
- InsertLabel = "insert"
- DeleteLabel = "delete"
- SearchLabel = "search"
- QueryLabel = "query"
-
+ InsertLabel = "insert"
+ DeleteLabel = "delete"
+ SearchLabel = "search"
+ QueryLabel = "query"
CacheHitLabel = "hit"
CacheMissLabel = "miss"
+ TimetickLabel = "timetick"
+ AllLabel = "all"
UnissuedIndexTaskLabel = "unissued"
InProgressIndexTaskLabel = "in-progress"
diff --git a/internal/metrics/proxy_metrics.go b/internal/metrics/proxy_metrics.go
index 6200fe8f1b..be623310f5 100644
--- a/internal/metrics/proxy_metrics.go
+++ b/internal/metrics/proxy_metrics.go
@@ -150,13 +150,13 @@ var (
Buckets: buckets, // unit: ms
}, []string{nodeIDLabelName})
- // ProxySyncTimeTick record Proxy synchronization timestamp statistics, differentiated by Channel.
- ProxySyncTimeTick = prometheus.NewGaugeVec(
+ // ProxySyncTimeTickLag record Proxy synchronization timestamp statistics, differentiated by Channel.
+ ProxySyncTimeTickLag = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
- Name: "sync_epoch_time",
- Help: "synchronized unix epoch per physical channel and default channel",
+ Name: "tt_lag_ms",
+ Help: "now time minus tt per physical channel",
}, []string{nodeIDLabelName, channelNameLabelName})
// ProxyApplyPrimaryKeyLatency record the latency that apply primary key.
@@ -247,7 +247,7 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyCacheStatsCounter)
registry.MustRegister(ProxyUpdateCacheLatency)
- registry.MustRegister(ProxySyncTimeTick)
+ registry.MustRegister(ProxySyncTimeTickLag)
registry.MustRegister(ProxyApplyPrimaryKeyLatency)
registry.MustRegister(ProxyApplyTimestampLatency)
diff --git a/internal/metrics/querynode_metrics.go b/internal/metrics/querynode_metrics.go
index 1931ec6e09..764b7986b2 100644
--- a/internal/metrics/querynode_metrics.go
+++ b/internal/metrics/querynode_metrics.go
@@ -17,6 +17,8 @@
package metrics
import (
+ "fmt"
+
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/prometheus/client_golang/prometheus"
)
@@ -32,6 +34,30 @@ var (
nodeIDLabelName,
})
+ QueryNodeConsumeTimeTickLag = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: milvusNamespace,
+ Subsystem: typeutil.QueryNodeRole,
+ Name: "consume_tt_lag_ms",
+ Help: "now time minus tt per physical channel",
+ }, []string{
+ nodeIDLabelName,
+ msgTypeLabelName,
+ collectionIDLabelName,
+ })
+
+ QueryNodeConsumerMsgCount = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: milvusNamespace,
+ Subsystem: typeutil.QueryNodeRole,
+ Name: "consume_msg_count",
+ Help: "count of consumed msg",
+ }, []string{
+ nodeIDLabelName,
+ msgTypeLabelName,
+ collectionIDLabelName,
+ })
+
QueryNodeNumPartitions = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
@@ -339,4 +365,27 @@ func RegisterQueryNode(registry *prometheus.Registry) {
registry.MustRegister(QueryNodeNumEntities)
registry.MustRegister(QueryNodeConsumeCounter)
registry.MustRegister(QueryNodeExecuteCounter)
+ registry.MustRegister(QueryNodeConsumerMsgCount)
+ registry.MustRegister(QueryNodeConsumeTimeTickLag)
+}
+
+func CleanupQueryNodeCollectionMetrics(nodeID int64, collectionID int64) {
+ for _, label := range []string{DeleteLabel, InsertLabel} {
+ QueryNodeConsumerMsgCount.
+ Delete(
+ prometheus.Labels{
+ nodeIDLabelName: fmt.Sprint(nodeID),
+ msgTypeLabelName: label,
+ collectionIDLabelName: fmt.Sprint(collectionID),
+ })
+
+ QueryNodeConsumeTimeTickLag.
+ Delete(
+ prometheus.Labels{
+ nodeIDLabelName: fmt.Sprint(nodeID),
+ msgTypeLabelName: label,
+ collectionIDLabelName: fmt.Sprint(collectionID),
+ })
+ }
+
}
diff --git a/internal/metrics/rootcoord_metrics.go b/internal/metrics/rootcoord_metrics.go
index aeae154329..61e89f7545 100644
--- a/internal/metrics/rootcoord_metrics.go
+++ b/internal/metrics/rootcoord_metrics.go
@@ -23,8 +23,8 @@ var (
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.RootCoordRole,
- Name: "sync_epoch_time",
- Help: "synchronized unix epoch per physical channel",
+ Name: "produce_tt_lag_ms",
+ Help: "now time minus tt per physical channel",
}, []string{channelNameLabelName})
RootCoordDDLReqCounter = prometheus.NewCounterVec(
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index 5e5b5ee28b..53974ce55c 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -291,8 +291,6 @@ func (node *Proxy) sendChannelsTimeTickLoop() {
maxTs := ts
for channel, ts := range stats {
- physicalTs, _ := tsoutil.ParseHybridTs(ts)
- metrics.ProxySyncTimeTick.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), channel).Set(float64(physicalTs))
channels = append(channels, channel)
tss = append(tss, ts)
if ts > maxTs {
@@ -311,8 +309,8 @@ func (node *Proxy) sendChannelsTimeTickLoop() {
Timestamps: tss,
DefaultTimestamp: maxTs,
}
- maxPhysicalTs, _ := tsoutil.ParseHybridTs(maxTs)
- metrics.ProxySyncTimeTick.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "default").Set(float64(maxPhysicalTs))
+ sub := tsoutil.SubByNow(maxTs)
+ metrics.ProxySyncTimeTickLag.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "default").Set(float64(sub))
status, err := node.rootCoord.UpdateChannelTimeTick(node.ctx, req)
if err != nil {
log.Warn("sendChannelsTimeTickLoop.UpdateChannelTimeTick", zap.Error(err))
diff --git a/internal/querynode/flow_graph_query_node.go b/internal/querynode/flow_graph_query_node.go
index 35f564f0c0..6f20eae5b4 100644
--- a/internal/querynode/flow_graph_query_node.go
+++ b/internal/querynode/flow_graph_query_node.go
@@ -22,6 +22,8 @@ import (
"fmt"
"time"
+ "github.com/milvus-io/milvus/internal/util/typeutil"
+
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/log"
@@ -71,7 +73,7 @@ func newQueryNodeFlowGraph(ctx context.Context,
flowGraph: flowgraph.NewTimeTickedFlowGraph(ctx1),
}
- dmStreamNode, err := q.newDmInputNode(ctx1, factory, collectionID, vchannel)
+ dmStreamNode, err := q.newDmInputNode(ctx1, factory, collectionID, vchannel, metrics.InsertLabel)
if err != nil {
return nil, err
}
@@ -137,7 +139,7 @@ func newQueryNodeDeltaFlowGraph(ctx context.Context,
flowGraph: flowgraph.NewTimeTickedFlowGraph(ctx1),
}
- dmStreamNode, err := q.newDmInputNode(ctx1, factory, collectionID, vchannel)
+ dmStreamNode, err := q.newDmInputNode(ctx1, factory, collectionID, vchannel, metrics.DeleteLabel)
if err != nil {
return nil, err
}
@@ -186,7 +188,8 @@ func newQueryNodeDeltaFlowGraph(ctx context.Context,
}
// newDmInputNode returns a new inputNode
-func (q *queryNodeFlowGraph) newDmInputNode(ctx context.Context, factory msgstream.Factory, collectionID UniqueID, vchannel Channel) (*flowgraph.InputNode, error) {
+
+func (q *queryNodeFlowGraph) newDmInputNode(ctx context.Context, factory msgstream.Factory, collectionID UniqueID, vchannel Channel, dataType string) (*flowgraph.InputNode, error) {
insertStream, err := factory.NewTtMsgStream(ctx)
if err != nil {
return nil, err
@@ -197,7 +200,8 @@ func (q *queryNodeFlowGraph) newDmInputNode(ctx context.Context, factory msgstre
maxQueueLength := Params.QueryNodeCfg.FlowGraphMaxQueueLength
maxParallelism := Params.QueryNodeCfg.FlowGraphMaxParallelism
name := fmt.Sprintf("dmInputNode-query-%d-%s", collectionID, vchannel)
- node := flowgraph.NewInputNode(insertStream, name, maxQueueLength, maxParallelism)
+ node := flowgraph.NewInputNode(insertStream, name, maxQueueLength, maxParallelism, typeutil.QueryNodeRole,
+ paramtable.GetNodeID(), collectionID, dataType)
return node, nil
}
@@ -267,4 +271,6 @@ func (q *queryNodeFlowGraph) close() {
zap.Int64("collectionID", q.collectionID),
zap.String("vchannel", q.vchannel),
)
+
+ metrics.CleanupQueryNodeCollectionMetrics(paramtable.GetNodeID(), q.collectionID)
}
diff --git a/internal/rootcoord/timeticksync.go b/internal/rootcoord/timeticksync.go
index b9e81f8660..6af9ccc445 100644
--- a/internal/rootcoord/timeticksync.go
+++ b/internal/rootcoord/timeticksync.go
@@ -318,9 +318,9 @@ func (t *timetickSync) sendTimeTickToChannel(chanNames []string, ts typeutil.Tim
return err
}
- physicalTs, _ := tsoutil.ParseHybridTs(ts)
+ sub := tsoutil.SubByNow(ts)
for _, chanName := range chanNames {
- metrics.RootCoordInsertChannelTimeTick.WithLabelValues(chanName).Set(float64(physicalTs))
+ metrics.RootCoordInsertChannelTimeTick.WithLabelValues(chanName).Set(float64(sub))
}
return nil
}
diff --git a/internal/util/flowgraph/input_node.go b/internal/util/flowgraph/input_node.go
index 136ed04e6d..e62a331110 100644
--- a/internal/util/flowgraph/input_node.go
+++ b/internal/util/flowgraph/input_node.go
@@ -17,9 +17,15 @@
package flowgraph
import (
+ "fmt"
"sync"
+ "github.com/milvus-io/milvus/internal/util/tsoutil"
+
+ "github.com/milvus-io/milvus/internal/util/typeutil"
+
"github.com/milvus-io/milvus/internal/log"
+ "github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/opentracing/opentracing-go"
@@ -30,9 +36,13 @@ import (
// InputNode is the entry point of flowgragh
type InputNode struct {
BaseNode
- inStream msgstream.MsgStream
- name string
- closeOnce sync.Once
+ inStream msgstream.MsgStream
+ name string
+ role string
+ nodeID int64
+ collectionID int64
+ dataType string
+ closeOnce sync.Once
}
// IsInputNode returns whether Node is InputNode
@@ -76,6 +86,28 @@ func (inNode *InputNode) Operate(in []Msg) []Msg {
if msgPack == nil {
return []Msg{}
}
+
+ sub := tsoutil.SubByNow(msgPack.EndTs)
+ if inNode.role == typeutil.QueryNodeRole {
+ metrics.QueryNodeConsumerMsgCount.
+ WithLabelValues(fmt.Sprint(inNode.nodeID), inNode.dataType, fmt.Sprint(inNode.collectionID)).
+ Inc()
+
+ metrics.QueryNodeConsumeTimeTickLag.
+ WithLabelValues(fmt.Sprint(inNode.nodeID), inNode.dataType, fmt.Sprint(inNode.collectionID)).
+ Set(float64(sub))
+ }
+
+ if inNode.role == typeutil.DataNodeRole {
+ metrics.DataNodeConsumeMsgCount.
+ WithLabelValues(fmt.Sprint(inNode.nodeID), inNode.dataType, fmt.Sprint(inNode.collectionID)).
+ Inc()
+
+ metrics.DataNodeConsumeTimeTickLag.
+ WithLabelValues(fmt.Sprint(inNode.nodeID), inNode.dataType, fmt.Sprint(inNode.collectionID)).
+ Set(float64(sub))
+ }
+
var spans []opentracing.Span
for _, msg := range msgPack.Msgs {
sp, ctx := trace.StartSpanFromContext(msg.TraceCtx())
@@ -101,14 +133,18 @@ func (inNode *InputNode) Operate(in []Msg) []Msg {
}
// NewInputNode composes an InputNode with provided MsgStream, name and parameters
-func NewInputNode(inStream msgstream.MsgStream, nodeName string, maxQueueLength int32, maxParallelism int32) *InputNode {
+func NewInputNode(inStream msgstream.MsgStream, nodeName string, maxQueueLength int32, maxParallelism int32, role string, nodeID int64, collectionID int64, dataType string) *InputNode {
baseNode := BaseNode{}
baseNode.SetMaxQueueLength(maxQueueLength)
baseNode.SetMaxParallelism(maxParallelism)
return &InputNode{
- BaseNode: baseNode,
- inStream: inStream,
- name: nodeName,
+ BaseNode: baseNode,
+ inStream: inStream,
+ name: nodeName,
+ role: role,
+ nodeID: nodeID,
+ collectionID: collectionID,
+ dataType: dataType,
}
}
diff --git a/internal/util/flowgraph/input_node_test.go b/internal/util/flowgraph/input_node_test.go
index 7b2baa8e0b..0f1d93ce3e 100644
--- a/internal/util/flowgraph/input_node_test.go
+++ b/internal/util/flowgraph/input_node_test.go
@@ -41,7 +41,7 @@ func TestInputNode(t *testing.T) {
produceStream.Produce(&msgPack)
nodeName := "input_node"
- inputNode := NewInputNode(msgStream, nodeName, 100, 100)
+ inputNode := NewInputNode(msgStream, nodeName, 100, 100, "", 0, 0, "")
defer inputNode.Close()
isInputNode := inputNode.IsInputNode()
@@ -64,7 +64,7 @@ func Test_NewInputNode(t *testing.T) {
nodeName := "input_node"
var maxQueueLength int32
var maxParallelism int32 = 100
- node := NewInputNode(nil, nodeName, maxQueueLength, maxParallelism)
+ node := NewInputNode(nil, nodeName, maxQueueLength, maxParallelism, "", 0, 0, "")
assert.NotNil(t, node)
assert.Equal(t, node.name, nodeName)
assert.Equal(t, node.maxQueueLength, maxQueueLength)
diff --git a/internal/util/flowgraph/node_test.go b/internal/util/flowgraph/node_test.go
index 86fc05bb93..3c27e1453e 100644
--- a/internal/util/flowgraph/node_test.go
+++ b/internal/util/flowgraph/node_test.go
@@ -74,7 +74,7 @@ func TestNodeCtx_Start(t *testing.T) {
produceStream.Produce(&msgPack)
nodeName := "input_node"
- inputNode := NewInputNode(msgStream, nodeName, 100, 100)
+ inputNode := NewInputNode(msgStream, nodeName, 100, 100, "", 0, 0, "")
node := &nodeCtx{
node: inputNode,
diff --git a/internal/util/tsoutil/tso.go b/internal/util/tsoutil/tso.go
index 9832ba5164..0873af2624 100644
--- a/internal/util/tsoutil/tso.go
+++ b/internal/util/tsoutil/tso.go
@@ -97,3 +97,10 @@ func AddPhysicalDurationOnTs(ts uint64, duration time.Duration) uint64 {
func NewTSOKVBase(client *clientv3.Client, tsoRoot, subPath string) *etcdkv.EtcdKV {
return etcdkv.NewEtcdKV(client, path.Join(tsoRoot, subPath))
}
+
+// SubByNow ts is a hybrid
+func SubByNow(ts uint64) int64 {
+ utcT, _ := ParseHybridTs(ts)
+ now := time.Now().UnixMilli()
+ return now - utcT
+}