diff --git a/internal/datacoord/compaction_task_meta.go b/internal/datacoord/compaction_task_meta.go index b055186dd4..6af1e13faf 100644 --- a/internal/datacoord/compaction_task_meta.go +++ b/internal/datacoord/compaction_task_meta.go @@ -63,7 +63,7 @@ func newCompactionTaskMeta(ctx context.Context, catalog metastore.DataCoordCatal ctx: ctx, catalog: catalog, compactionTasks: make(map[int64]map[int64]*datapb.CompactionTask, 0), - taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](1024, nil, time.Minute*60), + taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](32, nil, time.Minute*15), } if err := csm.reloadFromKV(); err != nil { return nil, err @@ -178,10 +178,6 @@ func (csm *compactionTaskMeta) DropCompactionTask(task *datapb.CompactionTask) e func (csm *compactionTaskMeta) TaskStatsJSON() string { tasks := csm.taskStats.Values() - if len(tasks) == 0 { - return "" - } - ret, err := json.Marshal(tasks) if err != nil { return "" diff --git a/internal/datacoord/compaction_task_meta_test.go b/internal/datacoord/compaction_task_meta_test.go index 48050fb640..ce3cb85a7d 100644 --- a/internal/datacoord/compaction_task_meta_test.go +++ b/internal/datacoord/compaction_task_meta_test.go @@ -111,7 +111,7 @@ func (suite *CompactionTaskMetaSuite) TestTaskStatsJSON() { // testing return empty string actualJSON := suite.meta.TaskStatsJSON() - suite.Equal("", actualJSON) + suite.Equal("[]", actualJSON) err := suite.meta.SaveCompactionTask(task1) suite.NoError(err) diff --git a/internal/datacoord/import_meta.go b/internal/datacoord/import_meta.go index da142157b5..b81de5950a 100644 --- a/internal/datacoord/import_meta.go +++ b/internal/datacoord/import_meta.go @@ -52,7 +52,7 @@ type importTasks struct { func newImportTasks() *importTasks { return &importTasks{ tasks: make(map[int64]ImportTask), - taskStats: expirable.NewLRU[UniqueID, ImportTask](4096, nil, time.Minute*60), + taskStats: expirable.NewLRU[UniqueID, ImportTask](64, nil, time.Minute*30), } } @@ -301,9 +301,6 @@ func (m *importMeta) RemoveTask(taskID int64) error { func (m *importMeta) TaskStatsJSON() string { tasks := m.tasks.listTaskStats() - if len(tasks) == 0 { - return "" - } ret, err := json.Marshal(tasks) if err != nil { diff --git a/internal/datacoord/import_meta_test.go b/internal/datacoord/import_meta_test.go index c61abbf69e..a9ed20f5eb 100644 --- a/internal/datacoord/import_meta_test.go +++ b/internal/datacoord/import_meta_test.go @@ -251,7 +251,7 @@ func TestTaskStatsJSON(t *testing.T) { assert.NoError(t, err) statsJSON := im.TaskStatsJSON() - assert.Equal(t, "", statsJSON) + assert.Equal(t, "[]", statsJSON) task1 := &importTask{ ImportTaskV2: &datapb.ImportTaskV2{ diff --git a/internal/datacoord/import_task.go b/internal/datacoord/import_task.go index fb2e59422a..719ad0035b 100644 --- a/internal/datacoord/import_task.go +++ b/internal/datacoord/import_task.go @@ -185,7 +185,7 @@ func (p *preImportTask) MarshalJSON() ([]byte, error) { NodeID: p.GetNodeID(), State: p.GetState().String(), Reason: p.GetReason(), - TaskType: "PreImportTask", + TaskType: p.GetType().String(), CreatedTime: p.GetCreatedTime(), CompleteTime: p.GetCompleteTime(), } @@ -231,7 +231,7 @@ func (t *importTask) MarshalJSON() ([]byte, error) { NodeID: t.GetNodeID(), State: t.GetState().String(), Reason: t.GetReason(), - TaskType: "ImportTask", + TaskType: t.GetType().String(), CreatedTime: t.GetCreatedTime(), CompleteTime: t.GetCompleteTime(), } diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go index dbc60101d0..e31f12ca2f 100644 --- a/internal/datacoord/index_meta.go +++ b/internal/datacoord/index_meta.go @@ -102,7 +102,7 @@ func newSegmentIndexBuildInfo() *segmentBuildInfo { // build ID -> segment index buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex), // build ID -> task stats - taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](1024, nil, time.Minute*60), + taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](64, nil, time.Minute*30), } } @@ -1075,10 +1075,6 @@ func (m *indexMeta) HasIndex(collectionID int64) bool { func (m *indexMeta) TaskStatsJSON() string { tasks := m.segmentBuildInfo.GetTaskStats() - if len(tasks) == 0 { - return "" - } - ret, err := json.Marshal(tasks) if err != nil { return "" diff --git a/internal/datacoord/index_meta_test.go b/internal/datacoord/index_meta_test.go index c5b2fa9c0c..9b4de11076 100644 --- a/internal/datacoord/index_meta_test.go +++ b/internal/datacoord/index_meta_test.go @@ -1543,7 +1543,7 @@ func TestBuildIndexTaskStatsJSON(t *testing.T) { } actualJSON := im.TaskStatsJSON() - assert.Equal(t, "", actualJSON) + assert.Equal(t, "[]", actualJSON) im.segmentBuildInfo.Add(si1) im.segmentBuildInfo.Add(si2) diff --git a/internal/datacoord/job_manager_test.go b/internal/datacoord/job_manager_test.go index 03ca4cf03a..a0d95e4cd5 100644 --- a/internal/datacoord/job_manager_test.go +++ b/internal/datacoord/job_manager_test.go @@ -104,7 +104,7 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() { allocator: alloc, tasks: make(map[int64]Task), meta: mt, - taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5), + taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*5), }, allocator: alloc, } diff --git a/internal/datacoord/metrics_info_test.go b/internal/datacoord/metrics_info_test.go index fe6114c067..1151dc70b5 100644 --- a/internal/datacoord/metrics_info_test.go +++ b/internal/datacoord/metrics_info_test.go @@ -325,7 +325,7 @@ func TestGetSyncTaskMetrics(t *testing.T) { mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)}) svr.cluster = mockCluster - expectedJSON := "" + expectedJSON := "null" actualJSON, err := svr.getSyncTaskJSON(ctx, req) assert.NoError(t, err) assert.Equal(t, expectedJSON, actualJSON) @@ -449,7 +449,7 @@ func TestGetSegmentsJSON(t *testing.T) { mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)}) svr.cluster = mockCluster - expectedJSON := "" + expectedJSON := "null" actualJSON, err := svr.getSegmentsJSON(ctx, req) assert.NoError(t, err) assert.Equal(t, expectedJSON, actualJSON) @@ -591,7 +591,7 @@ func TestGetChannelsJSON(t *testing.T) { svr.cluster = mockCluster svr.meta = &meta{channelCPs: newChannelCps()} - expectedJSON := "" + expectedJSON := "null" actualJSON, err := svr.getChannelsJSON(ctx, req) assert.NoError(t, err) assert.Equal(t, expectedJSON, actualJSON) diff --git a/internal/datacoord/task_scheduler.go b/internal/datacoord/task_scheduler.go index d176326ed0..5b26886630 100644 --- a/internal/datacoord/task_scheduler.go +++ b/internal/datacoord/task_scheduler.go @@ -91,7 +91,7 @@ func newTaskScheduler( handler: handler, indexEngineVersionManager: indexEngineVersionManager, allocator: allocator, - taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5), + taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15), } ts.reloadFromMeta() return ts diff --git a/internal/distributed/proxy/httpserver/handler.go b/internal/distributed/proxy/httpserver/handler.go index 6b781181d9..7bb8b5b20a 100644 --- a/internal/distributed/proxy/httpserver/handler.go +++ b/internal/distributed/proxy/httpserver/handler.go @@ -27,6 +27,9 @@ func (h *Handlers) RegisterRoutesTo(router gin.IRouter) { router.GET("/health", wrapHandler(h.handleGetHealth)) router.POST("/dummy", wrapHandler(h.handleDummy)) + router.GET("/databases", wrapHandler(h.handleListDatabases)) + router.GET("/database", wrapHandler(h.handleDescribeDatabases)) + router.POST("/collection", wrapHandler(h.handleCreateCollection)) router.DELETE("/collection", wrapHandler(h.handleDropCollection)) router.GET("/collection/existence", wrapHandler(h.handleHasCollection)) @@ -96,6 +99,24 @@ func (h *Handlers) handleDummy(c *gin.Context) (interface{}, error) { return h.proxy.Dummy(c, &req) } +func (h *Handlers) handleListDatabases(c *gin.Context) (interface{}, error) { + req := milvuspb.ListDatabasesRequest{} + err := shouldBind(c, &req) + if err != nil { + return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err) + } + return h.proxy.ListDatabases(c, &req) +} + +func (h *Handlers) handleDescribeDatabases(c *gin.Context) (interface{}, error) { + req := milvuspb.DescribeDatabaseRequest{} + err := shouldBind(c, &req) + if err != nil { + return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err) + } + return h.proxy.DescribeDatabase(c, &req) +} + func (h *Handlers) handleCreateCollection(c *gin.Context) (interface{}, error) { wrappedReq := WrappedCreateCollectionRequest{} err := shouldBind(c, &wrappedReq) diff --git a/internal/flushcommon/syncmgr/sync_manager.go b/internal/flushcommon/syncmgr/sync_manager.go index aa52b04a1d..bed92083b5 100644 --- a/internal/flushcommon/syncmgr/sync_manager.go +++ b/internal/flushcommon/syncmgr/sync_manager.go @@ -70,7 +70,7 @@ func NewSyncManager(chunkManager storage.ChunkManager) SyncManager { keyLockDispatcher: dispatcher, chunkManager: chunkManager, tasks: typeutil.NewConcurrentMap[string, Task](), - taskStats: expirable.NewLRU[string, Task](512, nil, time.Minute*15), + taskStats: expirable.NewLRU[string, Task](16, nil, time.Minute*15), } // setup config update watcher params.Watch(params.DataNodeCfg.MaxParallelSyncMgrTasks.Key, config.NewHandler("datanode.syncmgr.poolsize", syncMgr.resizeHandler)) diff --git a/internal/http/router.go b/internal/http/router.go index 2859704f48..b0e4463f04 100644 --- a/internal/http/router.go +++ b/internal/http/router.go @@ -75,6 +75,8 @@ const ( ClusterDependenciesPath = "/_cluster/dependencies" // HookConfigsPath is the path to get hook configurations. HookConfigsPath = "/_hook/configs" + // SlowQueryPath is the path to get slow queries metrics + SlowQueryPath = "/_cluster/slow_query" // QCDistPath is the path to get QueryCoord distribution. QCDistPath = "/_qc/dist" diff --git a/internal/http/webui/channels.html b/internal/http/webui/channels.html deleted file mode 100644 index 08c3b9c629..0000000000 --- a/internal/http/webui/channels.html +++ /dev/null @@ -1,158 +0,0 @@ - - - - - - - Milvus WebUI - Channels - - - - - - - - - - - -
- -
-
-
-
-

- Channel Checkpoints -

- - - - - - - - - - - - - - - - - - - - - - - - - - -
Channel NameCollection IDCheckpoint TsCheckpoint OffsetDatanode
channel1112022-11-11 12:00:00{ledgerID:1, entryID:1, batchIdx:0}datanode1
channel2222022-11-11 12:00:00{ledgerID:2, entryID:2, batchIdx:0}datanode1
- - -

- Watched Channels On Datanode -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Channel NameCollection IDConsume Rate/sLatencyTimeTickLagStateDatanode
channel12001150ms100mswatchingdatanode1
channel12001150ms100mswatcheddatanode2
- - -

- Watched Channels On QueryNode -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Channel NameCollection IDConsume Rate/sLatencyTimeTickLagStateDatanode
channel12001150ms100mswatchingquerynode1
channel22001150ms100mswatchingquerynode2
-
-
-
-
- -
- - - - \ No newline at end of file diff --git a/internal/http/webui/collections.html b/internal/http/webui/collections.html index 83cd46ed84..e6a9904ff7 100644 --- a/internal/http/webui/collections.html +++ b/internal/http/webui/collections.html @@ -23,110 +23,52 @@

- Database List + Database

- - - - - - - - - - - - - - - - - - -
Database IDNameCreate Time
1db12022-11-11 12:00:00
2db22022-11-11 12:00:00
+
+

+

+

- Collection List + Collection

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Collection IDCollection NamePartition Key CountPartition CountChannel CountSegment CountBinlog CountSize
1db1.coll1-fake111111
2db1.coll2-fake222222
+ +
-
+ +
  • + +
    • + + + +
    +
    +
    +
    + +
    +
    +
    +
    + +
    +

    +

    +
    + +
    @@ -137,12 +79,33 @@ $('#footer').load("footer.html"); }); - fetchData(MILVUS_URI + "/", clientInfos) + function searchCollections() { + const searchTerm = document.getElementById('databaseSearch').value; + let dbName = 'default'; + if (searchTerm !== '') { + dbName = searchTerm; + } + fetchCollections(dbName); + } + searchCollections() + + // TODO - Implement the following functions and support search with db name + // fetchData(MILVUS_URI + "/_collection/metrics", collectionRequest) + // .then(data => { + // collectionRequestsData = data; + // renderCollectionRequests(startPage, paginationSize); + // }) + // .catch(error => { + // handleError(error); + // }); + + fetchData(MILVUS_URI + "/databases", databases) .then(data => { - //TODO add collection render + databaseData = data; + renderDatabases(startPage, paginationSize) }) .catch(error => { - handleError(new Error("Unimplemented API")); + handleError(error); }); diff --git a/internal/http/webui/configs.html b/internal/http/webui/configs.html index 70b3b5fbf9..e257bc6895 100644 --- a/internal/http/webui/configs.html +++ b/internal/http/webui/configs.html @@ -24,7 +24,12 @@

    Milvus Configurations

    + +
    + +
    +

    Hook Configurations @@ -41,6 +46,12 @@ $('#footer').load("footer.html"); }); + function searchConfigs() { + const searchTerm = document.getElementById('searchInput').value; + currentPage = 0; // Reset to the first page on new search + renderConfigs(configData, searchTerm); + } + // load cluster configurations fetchData(MILVUS_URI + '/_cluster/configs', mconfigs) .then(data => { diff --git a/internal/http/webui/data_component.html b/internal/http/webui/data_component.html new file mode 100644 index 0000000000..195c316935 --- /dev/null +++ b/internal/http/webui/data_component.html @@ -0,0 +1,90 @@ + + + + + + + Milvus WebUI - Data Component + + + + + + + + + + + +
    + +
    +
    +
    +
    +

    Data Channels

    + + + + + + + + + + + + +
    Channel NameWatch StateNode IDLatest Time TickStart Watch TsCheckpoint Ts
    + +
    +

    +
    +
    + + +
    + + +

    Data Segments

    + + + + + + + + + + + + + +
    Segment IDCollection IDPartition IDChannelNum of RowsStateLevel
    + +
    +

    +
    +
    + + +
    + +
    +
    +
    +
    + +
    + + + + \ No newline at end of file diff --git a/internal/http/webui/header.html b/internal/http/webui/header.html index e87bd4472a..8d09f85781 100644 --- a/internal/http/webui/header.html +++ b/internal/http/webui/header.html @@ -11,17 +11,23 @@ Collections
  • - Segments + Query
  • - Channels + Data
  • Tasks
    • +
  • + Slow Requests +
  • Configurations
    • +
  • + Tools +
    • diff --git a/internal/http/webui/index.html b/internal/http/webui/index.html index 7cefd9cc94..641bd167f2 100644 --- a/internal/http/webui/index.html +++ b/internal/http/webui/index.html @@ -39,6 +39,9 @@
  • Runtime Metrics
    • +
  • + Requests +
    • @@ -52,7 +55,12 @@
    - + + +
    +
    +
    +

    Connected Clients @@ -86,6 +94,14 @@ handleError(error); }); + // fetchData(MILVUS_URI + "/_node/requests", nodeRequests) + // .then(data => { + // renderNodeRequests(data) + // }) + // .catch(error => { + // handleError(error); + // }); + fetchData(MILVUS_URI + "/_cluster/clients", clientInfos) .then(data => { renderClientsInfo(data); diff --git a/internal/http/webui/query_component.html b/internal/http/webui/query_component.html new file mode 100644 index 0000000000..4a557fe710 --- /dev/null +++ b/internal/http/webui/query_component.html @@ -0,0 +1,63 @@ + + + + + + + Milvus WebUI - Query Component + + + + + + + + + + + +
    + +
    +
    +
    +
    +

    Segments

    +
    +
    +
    +
    +
    + +

    Channels

    +
    +
    +
    +
    +
    + +

    Replicas

    +
    + +

    Resource Groups

    +
    +
    +
    +
    +
    + +
    + + + + + \ No newline at end of file diff --git a/internal/http/webui/query_target.html b/internal/http/webui/query_target.html new file mode 100644 index 0000000000..ddbcd2c1f6 --- /dev/null +++ b/internal/http/webui/query_target.html @@ -0,0 +1,86 @@ + + + + + + + Milvus WebUI - Query Component + + + + + + + + + + + +
    + +
    +
    +
    +
    +

    Target Segments

    + + + + + + + + + + + + + +
    Segment IDCollection IDPartition IDChannelNum of RowsStateTarget Scope
    +
    +
    +
    +
    + +

    Target Channels

    + + + + + + + + + + + + + + +
    Channel NameCollection IDNode IDVersionUnflushed SegmentsFlushed SegmentsDropped SegmentsTarget Scope
    + +
    +
    +
    +
    +
    + +
    +
    +
    + +
    + + + + + \ No newline at end of file diff --git a/internal/http/webui/segments.html b/internal/http/webui/segments.html deleted file mode 100644 index 580c143987..0000000000 --- a/internal/http/webui/segments.html +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - - Milvus WebUI - Segments - - - - - - - - - - - -
    - -
    -
    -
    -
    -

    - Loading Segments -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    segmentIDcollectionNameisIndexedsegmentSizequeryNode
    111coll1false6faked-querynode1
    111coll1false6faked-querynode2
    - - -

    - Releasing Segments -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    segmentIDcollectionNameisIndexedsegmentSizequeryNode
    111coll1false6faked-querynode1
    111coll1false6faked-querynode2
    - - -

    - Loaded Segments -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    segmentIDcollectionNameisIndexedsegmentSizequeryNode
    111coll1false6faked-querynode1
    111coll1116faked-querynode2
    - - - -

    - All Segments -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    segmentIDcollectionNamestaterowCountbinlog countbinlogs sizestatslogs sizedmlChannelleveldatanode
    111coll1Flushing11111111faked-channel-1L0faked-datanode1
    22222coll1Flushing11111111faked-channel-2L0faked-datanode2
    -
    -
    -
    -
    - -
    - - - - - \ No newline at end of file diff --git a/internal/http/webui/slow_requests.html b/internal/http/webui/slow_requests.html new file mode 100644 index 0000000000..68f6cdd2b0 --- /dev/null +++ b/internal/http/webui/slow_requests.html @@ -0,0 +1,58 @@ + + + + + + + Slow Requests + + + + + + + + + + + +
    + +
    +
    +
    +
    + + + +
    +

    + +
    +
    +
    +
    + +
    + + + + \ No newline at end of file diff --git a/internal/http/webui/static/css/style.css b/internal/http/webui/static/css/style.css index 9824ef0411..51f3ff8848 100644 --- a/internal/http/webui/static/css/style.css +++ b/internal/http/webui/static/css/style.css @@ -31,4 +31,10 @@ height: 1px; background-color: #000; margin: 40px 0; +} + +.table td.fit, +.table th.fit { + white-space: nowrap; + width: 1%; } \ No newline at end of file diff --git a/internal/http/webui/static/js/common.js b/internal/http/webui/static/js/common.js index 3f1bea9a1d..2757aabc73 100644 --- a/internal/http/webui/static/js/common.js +++ b/internal/http/webui/static/js/common.js @@ -27,18 +27,29 @@ toggleDebugMode(); const handleError = (error) => { console.error('Error fetching data:', error); - const errorMessage = encodeURIComponent(error.message || 'Unknown error'); - window.location.href = `5xx.html?error=${errorMessage}`; + // const errorMessage = encodeURIComponent(error.message || 'Unknown error'); + // window.location.href = `5xx.html?error=${errorMessage}`; }; -const fetchData = (url, localData) => { +const fetchData = (url, localData, kvParams) => { if (DEBUG_MODE) { return new Promise((resolve) => { resolve(JSON.parse(localData)); }); + } else if (kvParams && kvParams.length !== 0) { + return fetch(url, { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, + mode: 'no-cors', + body: JSON.stringify(kvParams) + }).then(response => response.json()) } else { - return fetch(url) - .then(response => response.json()) + return fetch(url).then(response => { + return response.json(); + }) } }; @@ -51,4 +62,18 @@ function getQueryParams() { params[decodeURIComponent(key)] = decodeURIComponent(value || ''); }); return params; +} + +function formatTimestamp(timestamp) { + const date = new Date(timestamp); // Convert timestamp to a Date object + // Format the date components + const year = date.getFullYear(); + const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are zero-indexed + const day = ('0' + date.getDate()).slice(-2); + const hours = ('0' + date.getHours()).slice(-2); + const minutes = ('0' + date.getMinutes()).slice(-2); + const seconds = ('0' + date.getSeconds()).slice(-2); + + // Return formatted date string + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}` } \ No newline at end of file diff --git a/internal/http/webui/static/js/mockdata.js b/internal/http/webui/static/js/mockdata.js index 9b07799002..1e21d99672 100644 --- a/internal/http/webui/static/js/mockdata.js +++ b/internal/http/webui/static/js/mockdata.js @@ -1,4 +1,4 @@ -var sysmetrics = `{ +const sysmetrics = `{ "nodes_info": [ { "identifier": 1, @@ -341,7 +341,40 @@ var sysmetrics = `{ ] }` -var clientInfos = `[ +const nodeRequests = ` +[ + { + "node_name": "querynode1", + "QPS": 0, + "read_request_count": 0, + "write_request_count": 0, + "delete_request_count": 0 + }, + { + "node_name": "datanode1", + "QPS": 0, + "read_request_count": 0, + "write_request_count": 0, + "delete_request_count": 0 + }, + { + "node_name": "indexnode1", + "QPS": 0, + "read_request_count": 0, + "write_request_count": 0, + "delete_request_count": 0 + }, + { + "node_name": "proxy1", + "QPS": 0, + "read_request_count": 0, + "write_request_count": 0, + "delete_request_count": 0 + } +] +` + +const clientInfos = `[ { "sdk_type": "python", "sdk_version": "1.0.0", @@ -364,7 +397,7 @@ var clientInfos = `[ } ]` -var dependencies = ` +const dependencies = ` { "metastore": { "health_status": true, @@ -386,7 +419,7 @@ var dependencies = ` } ` -var mconfigs = ` +const mconfigs = ` { "MILVUS_GIT_BUILD_TAGS": "v2.2-testing-20240702-811-g38211f2b81-dev", "MILVUS_GIT_COMMIT": "38211f2b81", @@ -412,7 +445,223 @@ var mconfigs = ` } `; -var qcTargets = ` +const collections =` +{ + "status": { + "error_code": "Success", + "reason": "" + }, + "collection_names": [ + "collection1", + "collection2", + "collection3", + "collection4", + "collection5", + "collection6", + "collection7", + "collection8", + "collection9", + "collection10" + ], + "collection_ids": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + ], + "created_timestamps": [ + 1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400 + ], + "created_utc_timestamps": [ + 1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400 + ], + "inMemory_percentages": [ + 100, 90, 80, 70, 60, 50, 40, 30, 20, 10 + ], + "query_service_available": [ + true, false, false, false, false, false, false, false, false, false + ] +} +` + +const collectionRequest = ` +[ + { + "collection_name": "collection1", + "search_QPS": 10, + "query_QPS": 5, + "write_throughput": 20, + "delete_QPS": 2 + }, + { + "collection_name": "collection2", + "search_QPS": 15, + "query_QPS": 7, + "write_throughput": 25, + "delete_QPS": 3 + }, + { + "collection_name": "collection3", + "search_QPS": 20, + "query_QPS": 10, + "write_throughput": 30, + "delete_QPS": 4 + }, + { + "collection_name": "collection4", + "search_QPS": 25, + "query_QPS": 12, + "write_throughput": 35, + "delete_QPS": 5 + }, + { + "collection_name": "collection5", + "search_QPS": 30, + "query_QPS": 15, + "write_throughput": 40, + "delete_QPS": 6 + }, + { + "collection_name": "collection6", + "search_QPS": 35, + "query_QPS": 17, + "write_throughput": 45, + "delete_QPS": 7 + }, + { + "collection_name": "collection7", + "search_QPS": 40, + "query_QPS": 20, + "write_throughput": 50, + "delete_QPS": 8 + }, + { + "collection_name": "collection8", + "search_QPS": 45, + "query_QPS": 22, + "write_throughput": 55, + "delete_QPS": 9 + }, + { + "collection_name": "collection9", + "search_QPS": 50, + "query_QPS": 25, + "write_throughput": 60, + "delete_QPS": 10 + }, + { + "collection_name": "collection10", + "search_QPS": 55, + "query_QPS": 27, + "write_throughput": 65, + "delete_QPS": 11 + } +] +` + +const describeCollectionResp = ` +{ + "status": { + "error_code": 0, + "reason": "Success" + }, + "schema": { + "name": "example_collection", + "description": "This is an example collection schema", + "fields": [ + { + "name": "field1", + "data_type": "INT64", + "is_primary_key": true, + "auto_id": false + }, + { + "name": "field2", + "data_type": "FLOAT", + "is_primary_key": false, + "auto_id": false + } + ] + }, + "collectionID": 12345, + "virtual_channel_names": ["vchan1", "vchan2"], + "physical_channel_names": ["pchan1", "pchan2"], + "created_timestamp": 1633036800, + "created_utc_timestamp": 1633036800, + "shards_num": 2, + "aliases": ["alias1", "alias2"], + "start_positions": [ + { + "key": "start_key", + "data": "start_data" + } + ], + "consistency_level": 0, + "collection_name": "example_collection", + "properties": [ + { + "key": "property_key", + "value": "property_value" + } + ], + "db_name": "example_db", + "num_partitions": 1, + "db_id": 1 +} +` + +const databases = ` +{ + "status": { + "error_code": "Success", + "reason": "" + }, + "db_names": [ + "database_1", + "database_2", + "database_3", + "database_4", + "database_5", + "database_6", + "database_7", + "database_8", + "database_9", + "database_10" + ], + "created_timestamp": [ + 1633036800, + 1633123200, + 1633209600, + 1633296000, + 1633382400, + 1633468800, + 1633555200, + 1633641600, + 1633728000, + 1633814400 + ], + "db_ids": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + ] +} +` + +const describeDatabaseResp = ` +{ + "status": { + "error_code": 0, + "reason": "Success" + }, + "db_name": "example_db", + "dbID": 1, + "created_timestamp": 1633036800, + "properties": [ + { + "key": "property_key", + "value": "property_value" + } + ] +} +` + +const qcCurrentTargets = ` [ { "collection_id": 1, @@ -442,7 +691,7 @@ var qcTargets = ` ], "resource_group": "rg1", "loaded_insert_row_count": 1000, - "mem_size": 2048, + "mem_size": 2048 } ], "dm_channels": [ @@ -451,10 +700,18 @@ var qcTargets = ` "version": 1, "collection_id": 1, "channel_name": "channel1", - "unflushed_segment_ids": [1], - "flushed_segment_ids": [2], - "dropped_segment_ids": [3], - "level_zero_segment_ids": [4], + "unflushed_segment_ids": [ + 1 + ], + "flushed_segment_ids": [ + 2 + ], + "dropped_segment_ids": [ + 3 + ], + "level_zero_segment_ids": [ + 4 + ], "partition_stats_versions": { "1": 1 } @@ -464,7 +721,44 @@ var qcTargets = ` ] ` -var qcDist =` +const qcNextTargets = ` +[ + { + "collection_id": 1, + "segments": [ + { + "segment_id": 2, + "collection_id": 1, + "partition_id": 1, + "channel": "channel2", + "num_of_rows": 1000, + "state": "Sealed", + "is_importing": false, + "compacted": false, + "level": "L0", + "is_sorted": true, + "node_id": 1, + "is_invisible": false, + "loaded_timestamp": 1633072800, + "index": [ + { + "field_id": 1, + "index_id": 1, + "build_id": 1, + "index_size": 1024, + "is_loaded": true + } + ], + "resource_group": "rg1", + "loaded_insert_row_count": 1000, + "mem_size": 2048 + } + ] + } +] +`; + +const qcDist = ` { "segments": [ { @@ -493,6 +787,9 @@ var qcDist =` "resource_group": "rg1", "loaded_insert_row_count": 1000, "mem_size": 2048, + "flushed_rows": 1000, + "sync_buffer_rows": 0, + "syncing_rows": 0 } ], "dm_channels": [ @@ -507,18 +804,24 @@ var qcDist =` "level_zero_segment_ids": [4], "partition_stats_versions": { "1": 1 - } + }, + "watch_state": "Healthy", + "start_watch_ts": 1633072800 } ], "leader_views": [ { - "node_id": 1, + "leader_id": 1, "collection_id": 1, - "channel_name": "channel1", - "segments": [ + "node_id": 1, + "channel": "channel1", + "version": 1, + "sealed_segments": [ { "segment_id": 1, + "collection_id": 1, "partition_id": 1, + "channel": "channel1", "num_of_rows": 1000, "state": "Sealed", "is_importing": false, @@ -540,52 +843,63 @@ var qcDist =` "resource_group": "rg1", "loaded_insert_row_count": 1000, "mem_size": 2048, + "flushed_rows": 1000, + "sync_buffer_rows": 0, + "syncing_rows": 0 } - ] + ], + "growing_segments": [], + "target_version": 1, + "num_of_growing_rows": 0, + "unserviceable_error": "" } ] } ` -var qcReplica = ` +const qcReplica = ` [ { "ID": 1, "CollectionID": 1, - "RWNodes": [1, 2], - "ResourceGroup": "rg1", - "RONodes": [3], - "ChannelToRWNodes": { + "rw_nodes": [1, 2], + "resource_group": "rg1", + "ro_nodes": [3], + "channel_to_rw_nodes": { "channel1": [1, 2] } }, { "ID": 2, "CollectionID": 2, - "RWNodes": [4, 5], - "ResourceGroup": "rg2", - "RONodes": [6], - "ChannelToRWNodes": { + "rw_nodes": [4, 5], + "resource_group": "rg2", + "ro_nodes": [6], + "channel_to_rw_nodes": { "channel2": [4, 5] } } ] -` +`; -var qcResourceGroup = ` +const qcResourceGroup = ` [ { - "Name": "rg1", - "Nodes": [1, 2] + "name": "rg1", + "nodes": [1, 2], + "cfg": { + "requests":{}, + "limits":{"node_num":1000000} + } }, { - "Name": "rg2", - "Nodes": [3, 4] + "name": "rg2", + "nodes": [3, 4] } ] -` +`; -var qcTasks = ` +const qcTasks = ` [ { "task_name": "balance_checker-ChannelTask[1]-ch1", @@ -598,7 +912,7 @@ var qcTasks = ` "type:Grow node id : 1 channel name:channel_1" ], "step": 1, - "reason": "some reason" + "reason": "" }, { "task_name": "index_checker-SegmentTask[2]-54321", @@ -611,7 +925,7 @@ var qcTasks = ` "type:Grow node id: 2 segment id:123 scope:DataScope_Streaming" ], "step": 2, - "reason": "another reason" + "reason": "" }, { "task_name": "leader_checker-LeaderSegmentTask[3]-1", @@ -627,9 +941,9 @@ var qcTasks = ` "reason": "yet another reason" } ] -` +`; -var qn_segments = ` +const qnSegments = ` [ { "segment_id": 1, @@ -656,7 +970,7 @@ var qn_segments = ` ], "resource_group": "rg1", "loaded_insert_row_count": 1000, - "mem_size": 2048, + "mem_size": 2048 }, { "segment_id": 2, @@ -683,12 +997,12 @@ var qn_segments = ` ], "resource_group": "rg2", "loaded_insert_row_count": 2000, - "mem_size": 4096, + "mem_size": 4096 } ] -` +`; -var qn_channels = ` +const qnChannels = ` [ { "name": "channel1", @@ -696,7 +1010,7 @@ var qn_channels = ` "assign_state": "assigned", "latest_time_tick": "2023-10-01 12:00:00", "node_id": 1, - "collection_id": 1, + "collection_id": 1 }, { "name": "channel2", @@ -704,26 +1018,39 @@ var qn_channels = ` "assign_state": "assigned", "latest_time_tick": "2023-10-01 12:05:00", "node_id": 2, - "collection_id": 2, + "collection_id": 2 } ] -` +`; -var dc_dist = ` +const dc_dist = ` { "segments": [ { "segment_id": 1, - "collection_id": 100, - "partition_id": 10, + "collection_id": 1, + "partition_id": 1, "channel": "channel1", "num_of_rows": 1000, - "state": "flushed", + "state": "Growing", "is_importing": false, "compacted": false, "level": "L1", "is_sorted": true, "node_id": 1 + }, + { + "segment_id": 3, + "collection_id": 2, + "partition_id": 2, + "channel": "channel2", + "num_of_rows": 2000, + "state": "Growing", + "is_importing": true, + "compacted": true, + "level": "L2", + "is_sorted": false, + "node_id": 2 } ], "dm_channels": [ @@ -737,12 +1064,23 @@ var dc_dist = ` "dropped_segment_ids": [7, 8, 9], "watch_state": "success", "start_watch_ts": 123456789 - } + }, + { + "node_id": 1, + "version": 1, + "collection_id": 100, + "channel_name": "channel3", + "unflushed_segment_ids": [1, 2, 3], + "flushed_segment_ids": [4, 5, 6], + "dropped_segment_ids": [7, 8, 9], + "watch_state": "to_watch", + "start_watch_ts": 123456789 + } ] } -` +`; -var dc_build_index_task = ` +const dc_build_index_task = ` [ { "index_id": 1, @@ -767,7 +1105,7 @@ var dc_build_index_task = ` } ]` -var dc_compaction_task = ` +const dc_compaction_task = ` [ { "plan_id": 1, @@ -795,7 +1133,7 @@ var dc_compaction_task = ` } ]` -var dn_sync_task = ` +const dn_sync_task = ` [ { "segment_id": 1, @@ -822,7 +1160,7 @@ var dn_sync_task = ` ] ` -var dc_import_task = ` +const dc_import_task = ` [ { "job_id": 1, @@ -840,9 +1178,9 @@ var dc_import_task = ` "task_id": 6, "collection_id": 7, "node_id": 8, - "state": "ImportTaskStateCompleted", + "state": "Completed", "reason": "", - "task_type": "Completed", + "task_type": "ImportTask", "created_time": "2023-10-01T00:00:00Z", "complete_time": "2023-10-01T01:00:00Z" }, @@ -860,7 +1198,7 @@ var dc_import_task = ` ] ` -var dn_segments = ` +const dn_segments = ` [ { "segment_id": 1, @@ -868,7 +1206,7 @@ var dn_segments = ` "partition_id": 1, "channel": "channel1", "num_of_rows": 1000, - "state": "active", + "state": "Growing", "is_importing": false, "compacted": false, "level": "L1", @@ -884,7 +1222,7 @@ var dn_segments = ` "partition_id": 2, "channel": "channel2", "num_of_rows": 2000, - "state": "inactive", + "state": "Sealed", "is_importing": true, "compacted": true, "level": "L2", @@ -897,7 +1235,7 @@ var dn_segments = ` ] ` -var dn_channels = ` +const dn_channels = ` [ { "name": "channel1", @@ -918,4 +1256,115 @@ var dn_channels = ` "check_point_ts": "2023-10-01 12:05:00" } ] -` \ No newline at end of file +` + +const slowQueries = `[ + { + "role": "proxy", + "database": "test_db", + "collection": "test_collection", + "partitions": "partition1,partition2", + "consistency_level": "Bounded", + "use_default_consistency": true, + "guarantee_timestamp": 123456789, + "duration": "1.1s", + "user": "test_user", + "query_params": { + "search_params": [ + { + "dsl": ["dsl1"], + "search_params": ["param2=value2"], + "nq": [10] + } + ], + "output_fields": "field1,field2" + }, + "type": "Search", + "trace_id": "729b10a6a7f32ddd7ab5c16dd30f60dc", + "time": "2024-11-05 08:14:05" + }, + { + "role": "proxy", + "database": "test_db", + "collection": "test_collection", + "partitions": "partition1,partition2", + "consistency_level": "Bounded", + "use_default_consistency": true, + "guarantee_timestamp": 123456789, + "duration": "1.2s", + "user": "test_user", + "query_params": { + "expr": "expr1", + "output_fields": "field1,field2" + }, + "type": "Query", + "trace_id": "232955b7f33b135708d34c3c761b57e7", + "time": "2024-11-05 08:14:05" + }, + { + "role": "proxy", + "database": "test_db", + "collection": "test_collection", + "partitions": "partition1,partition2", + "consistency_level": "Bounded", + "use_default_consistency": true, + "guarantee_timestamp": 123456789, + "duration": "1.3s", + "user": "test_user", + "query_params": { + "search_params": [ + { + "dsl": ["dsl2"], + "search_params": ["param3=value3"], + "nq": [20] + } + ], + "output_fields": "field3,field4" + }, + "type": "HybridSearch", + "trace_id": "3a4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j", + "time": "2024-11-05 08:14:05" + }, + { + "role": "proxy", + "database": "test_db", + "collection": "test_collection", + "partitions": "partition1,partition2", + "consistency_level": "Bounded", + "use_default_consistency": true, + "guarantee_timestamp": 123456789, + "duration": "1.4s", + "user": "test_user", + "query_params": { + "expr": "expr2", + "output_fields": "field5,field6" + }, + "type": "Query", + "trace_id": "4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k", + "time": "2024-11-05 08:14:05" + }, + { + "role": "proxy", + "database": "test_db", + "collection": "test_collection", + "partitions": "partition1,partition2", + "consistency_level": "Bounded", + "use_default_consistency": true, + "guarantee_timestamp": 123456789, + "duration": "1.5s", + "user": "test_user", + "query_params": { + "search_params": [ + { + "dsl": ["dsl3"], + "search_params": ["param4=value4"], + "nq": [30] + } + ], + "output_fields": "field7,field8" + }, + "type": "Search", + "trace_id": "5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k0l", + "time": "2024-11-05 08:14:05" + } +]`; diff --git a/internal/http/webui/static/js/render.js b/internal/http/webui/static/js/render.js index 7e5d35cf37..a7e55df81e 100644 --- a/internal/http/webui/static/js/render.js +++ b/internal/http/webui/static/js/render.js @@ -1,5 +1,13 @@ +const startPage = 0; +const paginationSize = 5; + function renderNodesMetrics(data) { + if (!data || !data.nodes_info || data.nodes_info.length === 0) { + document.getElementById("components").innerHTML = "No Found Components information" + return + } + let tableHTML = '' + 'Node Name' + 'CPU Usage' + @@ -35,15 +43,20 @@ function renderNodesMetrics(data) { } function renderComponentInfo(data) { + if (!data || !data.nodes_info || data.nodes_info.length === 0) { + document.getElementById("components").innerHTML = "No Found Components information" + return + } + let tableHTML = ` - - - Node Name - Node IP - Start Time - Node Status - - `; + + + Node Name + Node IP + Start Time + Node Status + + `; // Process each node's information data.nodes_info.forEach(node => { @@ -62,7 +75,260 @@ function renderComponentInfo(data) { document.getElementById("components").innerHTML = tableHTML; } +function renderNodeRequests(data) { + let tableHTML = '' + + 'Node Name' + + 'QPS' + + 'Read Request Count' + + 'Write Request Count' + + 'Delete Request Count' + + ''; + + tableHTML += ''; + data.forEach(node => { + tableHTML += ''; + tableHTML += `${node.node_name}`; + tableHTML += `${node.QPS}`; + tableHTML += `${node.read_request_count}`; + tableHTML += `${node.write_request_count}`; + tableHTML += `${node.delete_request_count}`; + tableHTML += ''; + }); + tableHTML += ''; + + document.getElementById('nodeRequests').innerHTML = tableHTML; +} + + +let databaseData = null; // Global variable to store fetched data +function renderDatabases(currentPage, rowsPerPage) { + if (!databaseData) { + console.error('No database data available'); + return; + } + + // Generate the HTML for the table with Bootstrap classes + let tableHTML = '' + + 'Name' + + 'ID' + + 'Created Timestamp' + + ''; + + tableHTML += ''; + + // Calculate start and end indices for pagination + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + + const totalCount = databaseData.db_names.length; + // Loop over page data to render only rows for the current page + for (let i = start; i < end && i < totalCount; i++) { + tableHTML += ''; + tableHTML += `${databaseData.db_names[i]}`; + tableHTML += `${databaseData.db_ids? databaseData.db_ids[i] : 0}`; + tableHTML += `${databaseData.created_timestamp? formatTimestamp(databaseData.created_timestamp[i]) : ''}`; + tableHTML += ''; + + // Hidden row for displaying collection details as JSON + tableHTML += ` + 
    Loading...
    + `; + } + tableHTML += ''; + + // Insert table HTML into the DOM + document.getElementById('database').innerHTML = tableHTML; + + // Display the total count + document.getElementById('db_totalCount').innerText = `Total Databases: ${totalCount}`; + + // Create pagination controls + const totalPages = Math.ceil(totalCount / rowsPerPage); + let paginationHTML = ''; + + // Insert pagination HTML into the DOM + document.getElementById('dbPaginationControls').innerHTML = paginationHTML; +} + +function describeDatabase(databaseName, rowIndex, type) { + fetchData(`${MILVUS_URI}/database?db_name=${databaseName}`, describeDatabaseResp) + .then(data => { + // Format data as JSON and insert into the designated row + const jsonFormattedData = JSON.stringify(data, null, 2); + document.getElementById(`${type}-json-details-${rowIndex}`).textContent = jsonFormattedData; + + // Toggle the visibility of the details row + const detailsRow = document.getElementById(`${type}-details-row-${rowIndex}`); + detailsRow.style.display = detailsRow.style.display === 'none' ? 'table-row' : 'none'; + }) + .catch(error => { + console.error('Error fetching collection details:', error); + document.getElementById(`${type}-json-details-${rowIndex}`).textContent = 'Failed to load collection details.'; + }); +} + +function describeCollection(databaseName, collectionName, rowIndex, type) { + fetchData(`${MILVUS_URI}/collection?db_name${databaseName}&&collection_name=${collectionName}`, describeCollectionResp) + .then(data => { + // Format data as JSON and insert into the designated row + const jsonFormattedData = JSON.stringify(data, null, 2); + document.getElementById(`${type}-json-details-${rowIndex}`).textContent = jsonFormattedData; + + // Toggle the visibility of the details row + const detailsRow = document.getElementById(`${type}-details-row-${rowIndex}`); + detailsRow.style.display = detailsRow.style.display === 'none' ? 'table-row' : 'none'; + }) + .catch(error => { + console.error('Error fetching collection details:', error); + document.getElementById(`${type}-json-details-${rowIndex}`).textContent = 'Failed to load collection details.'; + }); +} + +function fetchCollections(databaseName) { + fetchData(MILVUS_URI + `/collections?db_name=${databaseName}`, collections ) + .then(data => { + collectionsData = data; + renderCollections(databaseName, startPage, paginationSize) + }) + .catch(error => { + handleError(error); + }); +} + +let collectionsData = null; // Global variable to store fetched data +function renderCollections(databaseName, currentPage, rowsPerPage) { + let data = collectionsData; + if (!data) { + console.error('No collections data available'); + return; + } + let tableHTML = '' + + 'Name' + + 'Collection ID' + + 'Created Timestamp' + + 'Loaded Percentages' + + 'Queryable' + + ''; + + tableHTML += ''; + + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const totalCount = data.collection_names.length; + console.log(data) + for (let i = start; i < end && i < totalCount; i++) { + tableHTML += ''; + tableHTML += `${data.collection_names[i]}`; + tableHTML += `${data.collection_ids[i]}`; + tableHTML += `${formatTimestamp(data.created_utc_timestamps[i])}`; + tableHTML += `${data.inMemory_percentages? data.inMemory_percentages[i]: 'unknown'}`; + tableHTML += `${data.query_service_available? data.query_service_available[i] ? 'Yes' : 'No' : 'No'}`; + tableHTML += ''; + + // Hidden row for displaying collection details as JSON + tableHTML += ` + 
    Loading...
    + `; + } + tableHTML += ''; + + document.getElementById('collectionList').innerHTML = tableHTML; + document.getElementById('collection_totalCount').innerText = `Total Collections: ${totalCount}`; + + const totalPages = Math.ceil(totalCount / rowsPerPage); + let paginationHTML = ''; + document.getElementById('collectionPaginationControls').innerHTML = paginationHTML; +} + +let collectionRequestsData = null; // Global variable to store fetched data +function renderCollectionRequests(database, currentRequestPage, requestRowsPerPage) { + if (!collectionRequestsData) { + console.error('No collection requests data available'); + return; + } + + const data = collectionRequestsData; + let tableHTML = '' + + 'Collection Name' + + 'Search QPS' + + 'Query QPS' + + 'Insert Throughput(MB/s)' + + 'Delete QPS' + + ''; + + tableHTML += ''; + + const start = currentRequestPage * requestRowsPerPage; + const end = start + requestRowsPerPage; + const totalCount = data.length; + + for (let i = start; i < end && i < totalCount; i++) { + tableHTML += ''; + tableHTML += `${data[i].collection_name}`; + tableHTML += `${data[i].search_QPS}`; + tableHTML += `${data[i].query_QPS}`; + tableHTML += `${data[i].write_throughput}`; + tableHTML += `${data[i].delete_QPS}`; + tableHTML += ''; + + // Hidden row for displaying collection details as JSON + tableHTML += ` + 
    Loading...
    + `; + } + tableHTML += ''; + + document.getElementById('collectionRequests').innerHTML = tableHTML; + document.getElementById('collection_totalCount').innerText = `Total Collections: ${totalCount}`; + + const totalPages = Math.ceil(totalCount / requestRowsPerPage); + let paginationHTML = ''; + document.getElementById('collectionPaginationControls').innerHTML = paginationHTML; +} + function renderSysInfo(data) { + if (!data || data.length === 0 ) { + document.getElementById("sysInfo").innerHTML = "No Found Sys information" + return + } let tableHTML = '' + ' Attribute' + ' Value' + @@ -81,7 +347,7 @@ function renderSysInfo(data) { } function renderClientsInfo(data) { - if (data.length === 0 ) { + if (!data || data.length === 0 ) { document.getElementById("clients").innerHTML = "No clients connected" return } @@ -146,23 +412,92 @@ function readSysInfo(systemInfo) { return row } -function renderConfigs(obj) { +let configData = null; // Global variable to store the config object +let currentPage = 0; +const rowsPerPage = 10; + +function renderConfigs(obj, searchTerm = '') { + configData = obj; + + // Filter the data based on the search term + const filteredData = Object.keys(obj).filter(key => + key.toLowerCase().includes(searchTerm.toLowerCase()) || + String(obj[key]).toLowerCase().includes(searchTerm.toLowerCase()) + ).reduce((acc, key) => { + acc[key] = obj[key]; + return acc; + }, {}); + + // Calculate pagination variables + const totalCount = Object.keys(filteredData).length; + const totalPages = Math.ceil(totalCount / rowsPerPage); + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + + // Generate table header let tableHTML = '' + ' Attribute' + ' Value' + ''; - Object.keys(obj).forEach(function(prop) { - tableHTML += ''; + // Generate table rows based on current page and filtered data + tableHTML += ''; + const entries = Object.entries(filteredData).slice(start, end); + entries.forEach(([prop, value]) => { + tableHTML += ''; tableHTML += `${prop}`; - tableHTML += `${obj[prop]}`; - tableHTML += ``; - tableHTML += ''; + tableHTML += `${value}`; + tableHTML += ''; }); + tableHTML += ''; + + // Display the table document.getElementById('mConfig').innerHTML = tableHTML; + + // Display total count and pagination controls + let paginationHTML = '
    '; + + // Total count display + paginationHTML += `Total Configs: ${totalCount}`; + + // Pagination controls + paginationHTML += '
    '; + document.getElementById('paginationControls').innerHTML = paginationHTML; +} + +function changePage(page, searchTerm = '') { + currentPage = page; + renderConfigs(configData, searchTerm); } function renderDependencies(data) { + if (!data || data.length === 0 ) { + document.getElementById("3rdDependency").innerHTML = "No Found " + return + } + let tableHTML = '' + ' Sys Name' + ' Cluster Status' + @@ -190,3 +525,988 @@ function renderDependencies(data) { tableHTML += '' document.getElementById("3rdDependency").innerHTML = tableHTML; } + +function renderQCTasks(data) { + if (!data || data.length === 0 ) { + document.getElementById("qcTasks").innerHTML = "No Found QC Tasks" + return + } + let tableHTML = '' + + 'Task Name' + + 'Collection ID' + + 'Task Type' + + 'Task Status' + + 'Actions' + + ''; + + tableHTML += ''; + + data.forEach(task => { + let taskStatus = task.task_status; + if (taskStatus === 'failed') { + taskStatus = task.reason; + } + + tableHTML += ''; + tableHTML += `${task.task_name}`; + tableHTML += `${task.collection_id}`; + tableHTML += `${task.task_type}`; + tableHTML += `${taskStatus}`; + tableHTML += `${task.actions.join(', ')}`; + tableHTML += ''; + }); + + tableHTML += ''; + + document.getElementById('qcTasks').innerHTML = tableHTML; +} + +function renderBuildIndexTasks(data) { + if (!data || data.length === 0 ) { + document.getElementById("buildIndexTasks").innerHTML = "No Found Build Index Tasks" + return + } + let tableHTML = '' + + 'Index ID' + + 'Collection ID' + + 'Segment ID' + + 'Build ID' + + 'Index State' + + 'Index Size' + + 'Index Version' + + 'Create Time' + + ''; + + tableHTML += ''; + + data.forEach(task => { + let indexState = task.index_state; + if (indexState === 'Failed') { + indexState = task.fail_reason; + } + + tableHTML += ''; + tableHTML += `${task.index_id}`; + tableHTML += `${task.collection_id}`; + tableHTML += `${task.segment_id}`; + tableHTML += `${task.build_id}`; + tableHTML += `${indexState}`; + tableHTML += `${task.index_size}`; + tableHTML += `${task.index_version}`; + tableHTML += `${new Date(task.create_time * 1000).toLocaleString()}`; + tableHTML += ''; + }); + + tableHTML += ''; + + document.getElementById('buildIndexTasks').innerHTML = tableHTML; +} + +function renderCompactionTasks(data) { + if (!data || data.length === 0 ) { + document.getElementById("compactionTasks").innerHTML = "No Found Compaction Tasks" + return + } + + let tableHTML = '' + + 'Plan ID' + + 'Collection ID' + + 'Type' + + 'State' + + 'Start Time' + + 'End Time' + + 'Total Rows' + + 'Input Segments' + + 'Result Segments' + + ''; + + tableHTML += ''; + + data.forEach(task => { + let state = task.state; + if (state === 'Failed') { + state = task.fail_reason; + } + + tableHTML += ''; + tableHTML += `${task.plan_id}`; + tableHTML += `${task.collection_id}`; + tableHTML += `${task.type}`; + tableHTML += `${state}`; + tableHTML += `${new Date(task.start_time * 1000).toLocaleString()}`; + tableHTML += `${new Date(task.end_time * 1000).toLocaleString()}`; + tableHTML += `${task.total_rows}`; + tableHTML += `${task.input_segments.join(', ')}`; + tableHTML += `${task.result_segments.join(', ')}`; + tableHTML += ''; + }); + + tableHTML += ''; + + document.getElementById('compactionTasks').innerHTML = tableHTML; +} + +function renderImportTasks(data) { + if (!data || data.length === 0 ) { + document.getElementById("importTasks").innerHTML = "No Found Import Tasks" + return + } + let tableHTML = '' + + 'Job ID' + + 'Task ID' + + 'Collection ID' + + 'Node ID' + + 'State' + + 'Task Type' + + 'Created Time' + + 'Complete Time' + + ''; + + tableHTML += ''; + + data.forEach(task => { + let state = task.state; + if (state === 'Failed') { + state = task.reason; + } + + tableHTML += ''; + tableHTML += `${task.job_id}`; + tableHTML += `${task.task_id}`; + tableHTML += `${task.collection_id}`; + tableHTML += `${task.node_id}`; + tableHTML += `${state}`; + tableHTML += `${task.task_type}`; + tableHTML += `${new Date(task.created_time).toLocaleString()}`; + tableHTML += `${new Date(task.complete_time).toLocaleString()}`; + tableHTML += ''; + }); + + tableHTML += ''; + + document.getElementById('importTasks').innerHTML = tableHTML; +} + +function renderSyncTasks(data) { + if (!data || data.length === 0 ) { + document.getElementById("syncTasks").innerHTML = "No Found Sync Tasks" + return + } + let tableHTML = '' + + 'Segment ID' + + 'Batch Rows' + + 'Segment Level' + + 'Timestamp From' + + 'Timestamp To' + + 'Delta Row Count' + + 'Flush Size' + + 'Running Time' + + 'Node ID' + + ''; + + tableHTML += ''; + + data.forEach(task => { + tableHTML += ''; + tableHTML += `${task.segment_id}`; + tableHTML += `${task.batch_rows}`; + tableHTML += `${task.segment_level}`; + tableHTML += `${new Date(task.ts_from * 1000).toLocaleString()}`; + tableHTML += `${new Date(task.ts_to * 1000).toLocaleString()}`; + tableHTML += `${task.delta_row_count}`; + tableHTML += `${task.flush_size}`; + tableHTML += `${task.running_time}`; + tableHTML += `${task.node_id}`; + tableHTML += ''; + }); + + tableHTML += ''; + + document.getElementById('syncTasks').innerHTML = tableHTML; +} + +let dataChannels = null; // Store fetched merged dn_channels data and dc_dist data +function fetchAndRenderDataChannels() { + let dcDist = fetchData(MILVUS_URI + "/_dc/dist", dc_dist); + let dnChannels = fetchData(MILVUS_URI + "/_dn/channels", dn_channels); + Promise.all([dnChannels, dcDist]) + .then(([dnChannelsData, dcDistData]) => { + if (dnChannelsData && dcDistData) { + const { mergedChannels, notifications } = mergeChannels(dnChannelsData, dcDistData); + dataChannels = mergedChannels + renderChannels(mergedChannels, currentPage, rowsPerPage); + renderNotifications(notifications, 'notificationsDChannels'); + } + }) + .catch(error => { + handleError(error); + });; +} + +// Merge channels by matching names from dn_channels and dc_dist +function mergeChannels(dnChannels, dcDist) { + const merged = {}; + const notifications = []; // Store notifications for unmatched channels + // const dnChannelNames = new Set(dnChannels.map(channel => channel.name)); + const dcChannelNames = new Set(dcDist.dm_channels.map(dcChannel => dcChannel.channel_name)); + + // Add dn_channels to merged and mark channels missing in dc_dist + dnChannels.forEach(channel => { + channel.node_id = "datanode-" + channel.node_id; + const channelData = { ...channel }; + if (!dcChannelNames.has(channel.name)) { + channelData.notification = 'Not found in DataCoord'; + notifications.push(channelData); // Add to notifications + } + merged[channel.name] = channelData; + }); + + // Merge dc_dist channels or add new ones with notification if missing in dn_channels + dcDist.dm_channels.forEach(dcChannel => { + dcChannel.node_id = "datacoord-" + dcChannel.node_id; + const name = dcChannel.channel_name; + if (merged[name]) { + if (merged[name].watch_state !== "Healthy") { + dcChannel.watch_state = merged[name].watch_state + } + merged[name] = { ...merged[name], ...dcChannel }; + } else { + const channelData = { ...dcChannel, notification: 'Not found in DataNode' }; + notifications.push(channelData); // Add to notifications + merged[name] = channelData; + } + }); + + return { mergedChannels: Object.values(merged), notifications }; +} + +// Render the merged channels in a paginated table +function renderChannels(channels, currentPage, rowsPerPage) { + const table = document.getElementById("dataChannelsTableBody"); + table.innerHTML = ""; // Clear previous rows + + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedData = channels.slice(start, end); + + paginatedData.forEach(channel => { + const row = document.createElement("tr"); + + row.innerHTML = ` + ${channel.name || channel.channel_name} + ${channel.watch_state || "N/A"} + ${channel.node_id} + ${channel.latest_time_tick || "N/A"} + ${formatTimestamp(channel.start_watch_ts) || "N/A"} + ${channel.check_point_ts || "N/A"} + `; + table.appendChild(row); + }); + + // Update pagination info + const totalPages = Math.ceil(channels.length / rowsPerPage); + let paginationHTML = ''; + document.getElementById('dchannelPaginationControls').innerHTML = paginationHTML; + + document.getElementById('dchannel_totalCount').innerText = `Total Channels: ${channels.length}`; +} + +// Render notifications below the table +function renderNotifications(notifications, id) { + if (!notifications || notifications.length === 0) { + return + } + const notificationsContainer = document.getElementById(id); + notificationsContainer.innerHTML = ""; // Clear previous notifications + + notifications.forEach(channel => { + const notificationAlert = document.createElement("div"); + notificationAlert.className = "alert alert-warning"; + notificationAlert.role = "alert"; + + // Generate detailed information for each channel + const details = ` + Channel: ${channel.name || channel.channel_name} ${channel.notification}
    + `; + notificationAlert.innerHTML = details; + + notificationsContainer.appendChild(notificationAlert); + }); +} + +let dataSegments = null; // Store fetched merged dn_segments data and dc_dist data +const dataSegmentsStartPage = 0; +const dataSegmentsPaginationSize = 5; + +function fetchAndRenderDataSegments() { + let dcDist = fetchData(MILVUS_URI + "/_dc/dist", dc_dist); + let dnSegments = fetchData(MILVUS_URI + "/_dn/segments", dn_segments); + Promise.all([dnSegments, dcDist]) + .then(([dnSegmentsData, dcDistData]) => { + if (dnSegmentsData && dcDistData) { + const { mergedSegments, notifications } = mergeSegments(dnSegmentsData, dcDistData); + dataSegments = mergedSegments; + renderSegments(mergedSegments, currentPage, rowsPerPage); + renderNotifications(notifications, 'notificationsDSegments'); + } + }) + .catch(error => { + handleError(error); + }); +} + +// Merge segments by matching segment IDs from dn_segments and dc_dist +function mergeSegments(dnSegments, dcDist) { + const merged = {}; + const notifications = []; // Store notifications for unmatched segments + const dcSegmentIds = new Set(dcDist.segments.map(dcSegment => dcSegment.segment_id)); + + // Add dn_segments to merged and mark segments missing in dc_dist + dnSegments.forEach(segment => { + segment.node_id = "datanode-" + segment.node_id; + const segmentData = { ...segment }; + if (!dcSegmentIds.has(segment.segment_id)) { + segmentData.notification = 'Not found in DataCoord'; + notifications.push(segmentData); // Add to notifications + } + merged[segment.segment_id] = segmentData; + }); + + // Merge dc_dist segments or add new ones with notification if missing in dn_segments + dcDist.segments.forEach(dcSegment => { + dcSegment.node_id = "datacoord-" + dcSegment.node_id; + const id = dcSegment.segment_id; + if (merged[id]) { + merged[id] = { ...merged[id], ...dcSegment }; + } else { + const segmentData = { ...dcSegment, notification: 'Not found in DataNode' }; + if (dcSegment.state !== 'Dropped' && dcSegment.state !== 'Flushed'){ + notifications.push(segmentData); // Add to notifications + } + merged[id] = segmentData; + } + }); + + return { mergedSegments: Object.values(merged), notifications }; +} + +// Render the merged segments in a paginated table +function renderSegments(segments, currentPage, rowsPerPage) { + const table = document.getElementById("dataSegmentsTableBody"); + table.innerHTML = ""; // Clear previous rows + + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedData = segments.slice(start, end); + + paginatedData.forEach(segment => { + const row = document.createElement("tr"); + + const numRows = segment.state === "Growing" ? segment.flushed_rows : segment.num_of_rows; + row.innerHTML = ` + ${segment.segment_id} + ${segment.collection_id} + ${segment.partition_id} + ${segment.channel} + ${numRows || 'unknown'} + ${segment.state} + ${segment.level} + `; + table.appendChild(row); + }); + + // Update pagination info + const totalPages = Math.ceil(segments.length / rowsPerPage); + let paginationHTML = ''; + document.getElementById('dsegmentPaginationControls').innerHTML = paginationHTML; + + document.getElementById('dsegment_totalCount').innerText = `Total Segments: ${segments.length}`; +} + +function fetchAndRenderTargets(currentPage = startPage, rowsPerPage = rowsPerPage) { + let currentTargets = fetchData(MILVUS_URI + "/qc/target", qcCurrentTargets); + let nextTargets = fetchData(MILVUS_URI + "/_qc/target?target_scope=2", qcNextTargets); + + Promise.all([currentTargets, nextTargets]) + .then(([currentTargetsData, nextTargetsData]) => { + if (currentTargetsData && nextTargetsData) { + const mergedSegments = mergeTargetSegments(currentTargetsData, nextTargetsData); + const mergedChannels = mergeTargetChannels(currentTargetsData, nextTargetsData); + + renderTargetSegments(mergedSegments, currentPage, rowsPerPage); + renderTargetChannels(mergedChannels, currentPage, rowsPerPage); + } + }) + .catch(error => { + handleError(error); + }); +} + +function mergeTargetSegments(currentTargets, nextTargets) { + const merged = []; + + currentTargets.forEach(target => { + target.segments.forEach(segment => { + segment.targetScope = 'current'; + merged.push(segment); + }); + }); + + nextTargets.forEach(target => { + target.segments.forEach(segment => { + segment.targetScope = 'next'; + merged.push(segment); + }); + }); + + return merged; +} + +function mergeTargetChannels(currentTargets, nextTargets) { + const merged = []; + + currentTargets.forEach(target => { + if (target.dm_channels) { + target.dm_channels.forEach(channel => { + channel.targetScope = 'current'; + merged.push(channel); + }); + } + }); + + nextTargets.forEach(target => { + if (target.dm_channels) { + target.dm_channels.forEach(channel => { + channel.targetScope = 'next'; + merged.push(channel); + }); + } + }); + + return merged; +} + +function renderTargetSegments(segments, currentPage, rowsPerPage) { + const table = document.getElementById("dataSegmentsTableBody"); + table.innerHTML = ""; // Clear previous rows + + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedData = segments.slice(start, end); + + paginatedData.forEach(segment => { + const row = document.createElement("tr"); + + row.innerHTML = ` + ${segment.segment_id} + ${segment.collection_id} + ${segment.partition_id} + ${segment.channel} + ${segment.num_of_rows} + ${segment.state} + ${segment.targetScope} + `; + table.appendChild(row); + }); + + // Update pagination info + const totalPages = Math.ceil(segments.length / rowsPerPage); + let paginationHTML = ''; + document.getElementById('segmentPaginationControls').innerHTML = paginationHTML; + + document.getElementById('segment_totalCount').innerText = `Total Segments: ${segments.length}`; +} + +function renderTargetChannels(channels, currentPage, rowsPerPage) { + const table = document.getElementById("dataChannelsTableBody"); + table.innerHTML = ""; // Clear previous rows + + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedData = channels.slice(start, end); + + paginatedData.forEach(channel => { + const row = document.createElement("tr"); + + row.innerHTML = ` + ${channel.channel_name} + ${channel.collection_id} + ${channel.node_id} + ${channel.version} + ${channel.unflushed_segment_ids.join(', ')} + ${channel.flushed_segment_ids.join(', ')} + ${channel.dropped_segment_ids.join(', ')} + ${channel.targetScope} + `; + table.appendChild(row); + }); + + // Update pagination info + const totalPages = Math.ceil(channels.length / rowsPerPage); + let paginationHTML = ''; + document.getElementById('channelPaginationControls').innerHTML = paginationHTML; + document.getElementById('channel_totalCount').innerText = `Total Channels: ${channels.length}`; +} + +function renderSlowQueries(data) { + let tableHTML = '' + + 'Time' + + 'Trace ID' + + 'Request' + + 'User' + + 'Database' + + 'Collection' + + 'Parameters' + + 'Duration' + + ''; + + tableHTML += ''; + data.forEach(query => { + tableHTML += ''; + tableHTML += `${query.time}`; + tableHTML += `${query.trace_id}`; + tableHTML += `${query.type}`; + tableHTML += `${query.user}`; + tableHTML += `${query.database}`; + tableHTML += `${query.collection}`; + tableHTML += `${JSON.stringify(query.query_params)}`; + tableHTML += `${query.duration}`; + tableHTML += ''; + }); + tableHTML += ''; + + document.getElementById('slowQueries').innerHTML = tableHTML; + document.getElementById('slowQueriesTotalCount').innerText = `Total Slow Requests: ${data.length}`; +} + +const querySegmentsStartPage = 0; +const querySegmentsPaginationSize = 5; +var querySegments = null; +var queryChannels = null; + +function fetchAndRenderQuerySegmentsAndChannels(currentPage = querySegmentsStartPage, rowsPerPage = querySegmentsPaginationSize) { + let qcDistData = fetchData(MILVUS_URI + "/_qc/dist", qcDist); + let qcCurrentTargetsData = fetchData(MILVUS_URI + "/_qc/target", qcCurrentTargets); + let qcNextTargetsData = fetchData(MILVUS_URI + "/_qc/target?target_scope=2", qcNextTargets); + let qnSegmentsData = fetchData(MILVUS_URI + "/_qn/segments", qnSegments); + let qnChannelsData = fetchData(MILVUS_URI + "/_qn/channels", qnChannels); + + Promise.all([qcDistData, qcCurrentTargetsData, qcNextTargetsData, qnSegmentsData,qnChannelsData]) + .then(([qcDistData, qcCurrentTargetsData, qcNextTargetsData, qnSegmentsData,qnChannelsData]) => { + const mergedSegments = mergeQuerySegments(qcDistData, qcCurrentTargetsData, qcNextTargetsData, qnSegmentsData); + querySegments = mergedSegments + renderQuerySegments(mergedSegments, currentPage, rowsPerPage); + + document.getElementById('querySegmentsTotalCount').innerText = `Total Segments: ${mergedSegments.length}`; + renderQuerySegmentsPaginationControls(currentPage, Math.ceil(mergedSegments.length / rowsPerPage)); + + + const mergedChannels = mergeQueryChannels(qcDistData, qcCurrentTargetsData, qcNextTargetsData, qnChannelsData); + queryChannels = mergedChannels + renderQueryChannels(mergedChannels, currentPage, rowsPerPage); + + document.getElementById('queryChannelsTotalCount').innerText = `Total Channels: ${mergedChannels.length}`; + renderQueryChannelsPaginationControls(currentPage, Math.ceil(mergedChannels.length / rowsPerPage)); + }) + .catch(error => { + handleError(error); + }); +} + +function mergeQuerySegments(qcDist, qcCurrentTargets, qcNextTargets, qnSegments) { + const segments = []; + + const addSegment = (segment, from, leaderId = null) => { + const existingSegment = segments.find(s => s.segment_id === segment.segment_id); + if (existingSegment) { + if (from) { + existingSegment.from += ` | ${from}`; + } + if (leaderId) { + existingSegment.leader_id = leaderId; + } + if (segment.state && segment.state !== 'SegmentStateNone') { + existingSegment.state = segment.state; + } + } else { + segments.push({ ...segment, from, leader_id: leaderId }); + } + }; + + if (qcDist && qcDist.segments && qcDist.segments.length > 0) { + qcDist.segments.forEach(segment => { + addSegment(segment, 'DIST'); + }); + } + + if (qcCurrentTargets && qcCurrentTargets.length >0 ) { + qcCurrentTargets.forEach(target => { + target.segments.forEach(segment => { + addSegment(segment, 'CT'); + }); + }); + } + + if (qcCurrentTargets && qcCurrentTargets.length >0 ) { + qcCurrentTargets.forEach(target => { + target.segments.forEach(segment => { + addSegment(segment, 'NT'); + }); + }); + } + + if (qnSegments && qnSegments.length > 0) { + qnSegments.forEach(segment => { + addSegment(segment, 'QN'); + }); + } + + if (qcDist && qcDist.leader_views && qcDist.leader_views.length > 0) { + qcDist.leader_views.forEach(view => { + if (view.sealed_segments && view.sealed_segments.length > 0) { + view.sealed_segments.forEach(segment => { + addSegment(segment, null, view.leader_id); + }); + } + + if (view.growing_segments && view.growing_segments.length > 0) { + view.growing_segments.forEach(segment => { + addSegment(segment, null, view.leader_id); + }); + } + }); + } + + return segments; +} + +function renderQuerySegments(segments, currentPage, rowsPerPage) { + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedSegments = segments.slice(start, end); + + let tableHTML = ` + + + + + + + + + + + + + + `; + + paginatedSegments.forEach(segment => { + tableHTML += ` + + + + + + + + + + `; + }); + + tableHTML += ` + +
    Segment IDCollection IDLeader IDNode IDStateRowsFrom
    ${segment.segment_id}${segment.collection_id}${segment.leader_id || 'Not Found'}${segment.node_id}${segment.state}${segment.num_of_rows}${segment.from}
    + `; + + document.getElementById('querySegmentsTable').innerHTML = tableHTML; +} + + +function renderQuerySegmentsPaginationControls(currentPage, totalPages) { + let paginationHTML = ''; + document.getElementById('querySegmentsPagination').innerHTML = paginationHTML; +} + + +const queryChannelsStartPage = 0; +const queryChannelsPaginationSize = 5; +function renderQueryChannelsPaginationControls(currentPage, totalPages) { + let paginationHTML = ''; + document.getElementById('queryChannelsPagination').innerHTML = paginationHTML; +} + +function mergeQueryChannels(qcDist, qcCurrentTargets, qcNextTargets, qnChannels) { + const channels = []; + + const addChannel = (channel, from, leaderId = null) => { + const existingChannel = channels.find(c => c.name === channel.name || c.name === channel.channel_name); + if (existingChannel) { + if (from) { + existingChannel.from += ` | ${from}`; + } + if (leaderId) { + existingChannel.leader_id = leaderId; + } + + if (channel.watch_state && channel.watch_state !== 'Healthy' ) { + existingChannel.watch_state = channel.watch_state; + } + } else { + channels.push({ ...channel, from, leader_id: leaderId }); + } + }; + + if (qcDist && qcDist.dm_channels && qcDist.dm_channels.length > 0) { + qcDist.dm_channels.forEach(channel => { + addChannel(channel, 'DIST'); + }); + } + + if (qcCurrentTargets && qcCurrentTargets.length > 0) { + qcCurrentTargets.forEach(target => { + if (target.dm_channels) { + target.dm_channels.forEach(channel => { + addChannel(channel, 'CT'); + }); + } + }); + } + + if (qcNextTargets && qcNextTargets.length > 0) { + qcNextTargets.forEach(target => { + if (target.dm_channels) { + target.dm_channels.forEach(channel => { + addChannel(channel, 'NT'); + }); + } + }); + } + + if (qnChannels && qnChannels.length > 0) { + qnChannels.forEach(channel => { + addChannel(channel, 'QN'); + }); + } + + if (qcDist && qcDist.leader_views && qcDist.leader_views.length > 0) { + qcDist.leader_views.forEach(view => { + if (view.channel) { + addChannel({name: view.channel}, null, view.leader_id); + } + }); + } + + return channels; +} + +function renderQueryChannels(channels, currentPage, rowsPerPage) { + const start = currentPage * rowsPerPage; + const end = start + rowsPerPage; + const paginatedChannels = channels.slice(start, end); + + let tableHTML = ` + + + + + + + + + + + + + `; + + paginatedChannels.forEach(channel => { + tableHTML += ` + + + + + + + + + `; + }); + + tableHTML += ` + +
    Channel NameCollection IDLeader IDNode IDWatch StateFrom
    ${channel.name || channel.channel_name}${channel.collection_id}${channel.leader_id || 'Not Found'}${channel.node_id}${channel.watch_state}${channel.from}
    + `; + + document.getElementById('queryChannelsTable').innerHTML = tableHTML; +} + + +function fetchAndRenderReplicas() { + fetchData(MILVUS_URI + "/_qc/replica", qcReplica) + .then(data => { + renderReplicas(data); + }) + .catch(error => { + handleError(error); + }); +} + +function renderReplicas(replicas) { + if (!replicas || replicas.length === 0) { + return + } + + const table = document.getElementById('replicasTable'); + table.innerHTML = ''; // Clear the table + + // Render table headers + const headers = ['ID', 'Collection ID', 'RW Nodes', 'RO Nodes', 'Resource Group']; + const headerRow = document.createElement('tr'); + headers.forEach(header => { + const th = document.createElement('th'); + th.textContent = header; + headerRow.appendChild(th); + }); + table.appendChild(headerRow); + + // Render table rows + replicas.forEach(replica => { + const row = document.createElement('tr'); + row.innerHTML = ` + ${replica.ID} + ${replica.collectionID} + ${replica.rw_nodes? replica.rw_nodes.join(', '): ''} + ${replica.ro_nodes? replica.ro_nodes.join(', '): ''} + ${replica.resource_group} + `; + table.appendChild(row); + }); +} + +function fetchAndRenderResourceGroup() { + fetchData(MILVUS_URI + "/_qc/resource_group", qcResourceGroup) + .then(data => { + renderResourceGroup(data); + }) + .catch(error => { + console.error('Error fetching resource group data:', error); + }); +} + +function renderResourceGroup(data) { + if (!data || data.length === 0) { + return + } + + const table = document.getElementById('resourceGroupTable'); + table.innerHTML = ''; // Clear existing table content + + // Create table headers + const headerRow = document.createElement('tr'); + const headers = ['Name', 'Nodes', 'Cfg']; + headers.forEach(headerText => { + const header = document.createElement('th'); + header.textContent = headerText; + headerRow.appendChild(header); + }); + table.appendChild(headerRow); + + // Populate table rows with data + data.forEach(resourceGroup => { + const row = document.createElement('tr'); + + const nameCell = document.createElement('td'); + nameCell.textContent = resourceGroup.name; + row.appendChild(nameCell); + + const nodesCell = document.createElement('td'); + nodesCell.textContent = resourceGroup.nodes.join(', '); + row.appendChild(nodesCell); + + const cfgCell = document.createElement('td'); + cfgCell.textContent = resourceGroup.cfg? JSON.stringify(resourceGroup.cfg) : ''; + row.appendChild(cfgCell) + + table.appendChild(row); + }); +} \ No newline at end of file diff --git a/internal/http/webui/tasks.html b/internal/http/webui/tasks.html index 58f2c8834c..6565045e16 100644 --- a/internal/http/webui/tasks.html +++ b/internal/http/webui/tasks.html @@ -25,123 +25,29 @@

    QueryCoord Tasks

    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Task IDSourceActionsTypeState
    1indexCheckerquerynode1querynode2queued
    2loadChannelquerynode1querynode2queued
    +

    Compaction Tasks

    - - - - - - - - - - - - - - - - - - - - - - - -
    Task IDPlanStateDatanode
    1fake-planrunningdatanode1
    2fake-planrunningdatanode2
    - -

    - Flush Tasks -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Task IDSegment IDInsertion Size(MB)Deletion Size(MB)StateDatanode
    11111105runningdatanode1
    2222255runningdatanode2
    +

    Index Build Tasks

    - - - - - - - - - - - - - - - - - - - - - - - +
    Task IDPlanStateIndexnode
    1fake-planrunningdatanode1
    2fake-planrunningdatanode2
    + +

    + Import Tasks +

    +
    + +

    + Sync Task +

    +
    +
    @@ -156,13 +62,47 @@ }); document.addEventListener("DOMContentLoaded", function() { - fetchData(MILVUS_URI + "/", sysmetrics) + fetchData(MILVUS_URI + "/_qc/tasks", qcTasks) .then(data => { - //TODO add tasks render + renderQCTasks(data) }) .catch(error => { - handleError(new Error("Unimplemented API")); + handleError(error); }); + + fetchData(MILVUS_URI + "/_dc/tasks/compaction", dc_compaction_task) + .then(data => { + renderCompactionTasks(data) + }) + .catch(error => { + handleError(error); + }); + + fetchData(MILVUS_URI + "/_dc/tasks/build_index", dc_build_index_task) + .then(data => { + renderBuildIndexTasks(data) + }) + .catch(error => { + handleError(error); + }); + + fetchData(MILVUS_URI + "/_dc/tasks/import", dc_import_task) + .then(data => { + renderImportTasks(data) + }) + .catch(error => { + handleError(error); + }); + + fetchData(MILVUS_URI + "/_dn/tasks/sync", dn_sync_task) + .then(data => { + renderSyncTasks(data) + }) + .catch(error => { + handleError(error); + }); + + }); diff --git a/internal/http/webui/tools.html b/internal/http/webui/tools.html new file mode 100644 index 0000000000..3da83ff13e --- /dev/null +++ b/internal/http/webui/tools.html @@ -0,0 +1,52 @@ + + + + + + + tools + + + + + + + + + + + +
    + +
    +
    +
    +
    +
    + + +
    +
    + Pprof +
    +
    +
    +
    + Memory Data Visualization +
    +
    +
    +
    +
    +
    + +
    + + + + \ No newline at end of file diff --git a/internal/proxy/http_req_impl.go b/internal/proxy/http_req_impl.go index b0d68ae82c..1cf43f2f4a 100644 --- a/internal/proxy/http_req_impl.go +++ b/internal/proxy/http_req_impl.go @@ -118,6 +118,20 @@ func getDependencies(c *gin.Context) { c.Data(http.StatusOK, contentType, ret) } +func getSlowQuery(node *Proxy) gin.HandlerFunc { + return func(c *gin.Context) { + slowQueries := node.slowQueries.Values() + ret, err := json.Marshal(slowQueries) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + mhttp.HTTPReturnMessage: err.Error(), + }) + return + } + c.Data(http.StatusOK, contentType, ret) + } +} + // buildReqParams fetch all parameters from query parameter of URL, add them into a map data structure. // put key and value from query parameter into map, concatenate values with separator if values size is greater than 1 func buildReqParams(c *gin.Context, metricsType string) map[string]interface{} { diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index 7c74600b88..0157c63c23 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -30,6 +30,7 @@ import ( "github.com/samber/lo" "github.com/tidwall/gjson" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" @@ -3020,6 +3021,14 @@ func (node *Proxy) search(ctx context.Context, request *milvuspb.SearchRequest, strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.SearchLabel, ).Inc() + user, _ := GetCurUserFromContext(ctx) + traceID := "" + if sp != nil { + traceID = sp.SpanContext().TraceID().String() + } + if node.slowQueries != nil { + node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(request, user, span, traceID)) + } } }() @@ -3230,6 +3239,14 @@ func (node *Proxy) hybridSearch(ctx context.Context, request *milvuspb.HybridSea strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.HybridSearchLabel, ).Inc() + user, _ := GetCurUserFromContext(ctx) + traceID := "" + if sp != nil { + traceID = sp.SpanContext().TraceID().String() + } + if node.slowQueries != nil { + node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(newSearchReq, user, span, traceID)) + } } }() @@ -3471,7 +3488,7 @@ func (node *Proxy) Flush(ctx context.Context, request *milvuspb.FlushRequest) (* } // Query get the records by primary keys. -func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryResults, error) { +func (node *Proxy) query(ctx context.Context, qt *queryTask, sp trace.Span) (*milvuspb.QueryResults, error) { request := qt.request method := "Query" @@ -3513,6 +3530,15 @@ func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryRes strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.QueryLabel, ).Inc() + user, _ := GetCurUserFromContext(ctx) + traceID := "" + if sp != nil { + traceID = sp.SpanContext().TraceID().String() + } + + if node.slowQueries != nil { + node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithQueryRequest(request, user, span, traceID)) + } } }() @@ -3628,7 +3654,7 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (* request.GetCollectionName(), ).Inc() - res, err := node.query(ctx, qt) + res, err := node.query(ctx, qt, sp) if err != nil || !merr.Ok(res.Status) { return res, err } @@ -6484,6 +6510,9 @@ func (node *Proxy) RegisterRestRouter(router gin.IRouter) { // Hook request that executed by proxy router.GET(http.HookConfigsPath, getConfigs(paramtable.GetHookParams().GetAll())) + // Slow query request that executed by proxy + router.GET(http.SlowQueryPath, getSlowQuery(node)) + // QueryCoord requests that are forwarded from proxy router.GET(http.QCTargetPath, getQueryComponentMetrics(node, metricsinfo.QueryTarget)) router.GET(http.QCDistPath, getQueryComponentMetrics(node, metricsinfo.QueryDist)) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index acb4222294..8486526dd6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -26,6 +26,7 @@ import ( "time" "github.com/cockroachdb/errors" + "github.com/hashicorp/golang-lru/v2/expirable" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/atomic" "go.uber.org/zap" @@ -130,6 +131,8 @@ type Proxy struct { // delete rate limiter enableComplexDeleteLimit bool + + slowQueries *expirable.LRU[Timestamp, *metricsinfo.SlowQuery] } // NewProxy returns a Proxy struct. @@ -152,6 +155,7 @@ func NewProxy(ctx context.Context, factory dependency.Factory) (*Proxy, error) { lbPolicy: lbPolicy, resourceManager: resourceManager, replicateStreamManager: replicateStreamManager, + slowQueries: expirable.NewLRU[Timestamp, *metricsinfo.SlowQuery](20, nil, time.Minute*15), } node.UpdateStateCode(commonpb.StateCode_Abnormal) expr.Register("proxy", node) diff --git a/internal/proxy/task_search.go b/internal/proxy/task_search.go index 511bbf7485..3dc48cfe95 100644 --- a/internal/proxy/task_search.go +++ b/internal/proxy/task_search.go @@ -9,6 +9,7 @@ import ( "github.com/cockroachdb/errors" "github.com/samber/lo" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -734,7 +735,7 @@ func (t *searchTask) PostExecute(ctx context.Context) error { t.fillInFieldInfo() if t.requery { - err = t.Requery() + err = t.Requery(sp) if err != nil { log.Warn("failed to requery", zap.Error(err)) return err @@ -819,7 +820,7 @@ func (t *searchTask) estimateResultSize(nq int64, topK int64) (int64, error) { //return int64(sizePerRecord) * nq * topK, nil } -func (t *searchTask) Requery() error { +func (t *searchTask) Requery(span trace.Span) error { queryReq := &milvuspb.QueryRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_Retrieve, @@ -864,7 +865,7 @@ func (t *searchTask) Requery() error { fastSkip: true, reQuery: true, } - queryResult, err := t.node.(*Proxy).query(t.ctx, qt) + queryResult, err := t.node.(*Proxy).query(t.ctx, qt, span) if err != nil { return err } diff --git a/internal/proxy/task_search_test.go b/internal/proxy/task_search_test.go index 5409affb93..606e277f39 100644 --- a/internal/proxy/task_search_test.go +++ b/internal/proxy/task_search_test.go @@ -2744,7 +2744,7 @@ func TestSearchTask_Requery(t *testing.T) { node: node, } - err := qt.Requery() + err := qt.Requery(nil) assert.NoError(t, err) assert.Len(t, qt.result.Results.FieldsData, 2) for _, field := range qt.result.Results.FieldsData { @@ -2773,7 +2773,7 @@ func TestSearchTask_Requery(t *testing.T) { node: node, } - err := qt.Requery() + err := qt.Requery(nil) t.Logf("err = %s", err) assert.Error(t, err) }) @@ -2807,7 +2807,7 @@ func TestSearchTask_Requery(t *testing.T) { node: node, } - err := qt.Requery() + err := qt.Requery(nil) t.Logf("err = %s", err) assert.Error(t, err) }) diff --git a/internal/querycoordv2/meta/dist_manager.go b/internal/querycoordv2/meta/dist_manager.go index 048b7aac87..d5b54da49b 100644 --- a/internal/querycoordv2/meta/dist_manager.go +++ b/internal/querycoordv2/meta/dist_manager.go @@ -48,10 +48,6 @@ func (dm *DistributionManager) GetDistributionJSON() string { channels := dm.GetChannelDist() leaderView := dm.GetLeaderView() - if len(segments) == 0 && len(channels) == 0 && len(leaderView) == 0 { - return "" - } - dist := &metricsinfo.QueryCoordDist{ Segments: segments, DMChannels: channels, diff --git a/internal/querycoordv2/server.go b/internal/querycoordv2/server.go index 61989a0e99..4704848773 100644 --- a/internal/querycoordv2/server.go +++ b/internal/querycoordv2/server.go @@ -200,11 +200,16 @@ func (s *Server) registerMetricsRequest() { } QueryDistAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { - return s.targetMgr.GetTargetJSON(meta.CurrentTarget), nil + return s.dist.GetDistributionJSON(), nil } QueryTargetAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { - return s.dist.GetDistributionJSON(), nil + scope := meta.CurrentTarget + v := jsonReq.Get(metricsinfo.MetricRequestParamTargetScopeKey) + if v.Exists() { + scope = meta.TargetScope(v.Int()) + } + return s.targetMgr.GetTargetJSON(scope), nil } QueryReplicasAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { diff --git a/internal/querycoordv2/task/scheduler.go b/internal/querycoordv2/task/scheduler.go index 7295503d6d..0071cdae48 100644 --- a/internal/querycoordv2/task/scheduler.go +++ b/internal/querycoordv2/task/scheduler.go @@ -202,7 +202,7 @@ func NewScheduler(ctx context.Context, channelTasks: make(map[replicaChannelIndex]Task), processQueue: newTaskQueue(), waitQueue: newTaskQueue(), - taskStats: expirable.NewLRU[UniqueID, Task](512, nil, time.Minute*30), + taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15), } } @@ -574,10 +574,6 @@ func (scheduler *taskScheduler) GetSegmentTaskNum() int { // GetTasksJSON returns the JSON string of all tasks. // the task stats object is thread safe and can be accessed without lock func (scheduler *taskScheduler) GetTasksJSON() string { - if scheduler.taskStats.Len() == 0 { - return "" - } - tasks := scheduler.taskStats.Values() ret, err := json.Marshal(tasks) if err != nil { diff --git a/pkg/go.mod b/pkg/go.mod index 6a44630938..417d0277cf 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -26,6 +26,7 @@ require ( github.com/shirou/gopsutil/v3 v3.22.9 github.com/sirupsen/logrus v1.9.0 github.com/spaolacci/murmur3 v1.1.0 + github.com/spf13/cast v1.3.0 github.com/streamnative/pulsarctl v0.5.0 github.com/stretchr/testify v1.9.0 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c @@ -55,7 +56,7 @@ require ( google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.28.6 ) @@ -169,7 +170,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/pkg/go.sum b/pkg/go.sum index 583d5b7198..acbde0f955 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -655,6 +655,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= diff --git a/pkg/util/metricsinfo/metric_request.go b/pkg/util/metricsinfo/metric_request.go index 63504ab671..212177bfde 100644 --- a/pkg/util/metricsinfo/metric_request.go +++ b/pkg/util/metricsinfo/metric_request.go @@ -88,6 +88,8 @@ const ( // MetricRequestParamVerboseKey as a request parameter decide to whether return verbose value MetricRequestParamVerboseKey = "verbose" + + MetricRequestParamTargetScopeKey = "target_scope" ) type MetricsRequestAction func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) diff --git a/pkg/util/metricsinfo/metrics_info.go b/pkg/util/metricsinfo/metrics_info.go index 3393b77196..7cc3f5a639 100644 --- a/pkg/util/metricsinfo/metrics_info.go +++ b/pkg/util/metricsinfo/metrics_info.go @@ -70,6 +70,34 @@ const ( MilvusUsedGoVersion = "MILVUS_USED_GO_VERSION" ) +type SearchParams struct { + DSL []string `json:"dsl,omitempty"` + SearchParams []string `json:"search_params,omitempty"` + NQ []int64 `json:"nq,omitempty"` +} + +type QueryParams struct { + SearchParams []*SearchParams `json:"search_params,omitempty"` + Expr string `json:"expr,omitempty"` + OutputFields string `json:"output_fields,omitempty"` +} + +type SlowQuery struct { + Time string `json:"time,omitempty"` + Role string `json:"role,omitempty"` + Database string `json:"database,omitempty"` + Collection string `json:"collection,omitempty"` + Partitions string `json:"partitions,omitempty"` + ConsistencyLevel string `json:"consistency_level,omitempty"` + UseDefaultConsistency bool `json:"use_default_consistency,omitempty"` + GuaranteeTimestamp uint64 `json:"guarantee_timestamp,omitempty"` + Duration string `json:"duration,omitempty"` + User string `json:"user,omitempty"` + QueryParams *QueryParams `json:"query_params,omitempty"` + Type string `json:"type,omitempty"` + TraceID string `json:"trace_id,omitempty"` +} + type DmChannel struct { NodeID int64 `json:"node_id,omitempty"` Version int64 `json:"version,omitempty"` @@ -109,7 +137,6 @@ type Segment struct { FlushedRows int64 `json:"flushed_rows,omitempty"` SyncBufferRows int64 `json:"sync_buffer_rows,omitempty"` SyncingRows int64 `json:"syncing_rows,omitempty"` - // TODO add checkpoints } type SegmentIndex struct { @@ -127,15 +154,16 @@ type QueryCoordTarget struct { } type LeaderView struct { - LeaderID int64 `json:"leader_id"` - CollectionID int64 `json:"collection_id"` - Channel string `json:"channel"` - Version int64 `json:"version"` - SealedSegments []*Segment `json:"sealed_segments"` - GrowingSegments []*Segment `json:"growing_segments"` - TargetVersion int64 `json:"target_version"` - NumOfGrowingRows int64 `json:"num_of_growing_rows"` - UnServiceableError string `json:"unserviceable_error"` + LeaderID int64 `json:"leader_id,omitempty"` + CollectionID int64 `json:"collection_id,omitempty"` + NodeID int64 `json:"node_id,omitempty"` + Channel string `json:"channel,omitempty"` + Version int64 `json:"version,omitempty"` + SealedSegments []*Segment `json:"sealed_segments,omitempty"` + GrowingSegments []*Segment `json:"growing_segments,omitempty"` + TargetVersion int64 `json:"target_version,omitempty"` + NumOfGrowingRows int64 `json:"num_of_growing_rows,omitempty"` + UnServiceableError string `json:"unserviceable_error,omitempty"` } type QueryCoordDist struct { diff --git a/pkg/util/metricsinfo/utils.go b/pkg/util/metricsinfo/utils.go index 93e654dfd5..11cb9f644c 100644 --- a/pkg/util/metricsinfo/utils.go +++ b/pkg/util/metricsinfo/utils.go @@ -14,10 +14,15 @@ package metricsinfo import ( "encoding/json" "os" + "strings" + "time" "go.uber.org/zap" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/util/typeutil" ) // FillDeployMetricsWithEnv fill deploy metrics with env. @@ -34,10 +39,6 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) { return "", err } - if len(metrics) == 0 { - return "", nil - } - bs, err := json.Marshal(metrics) if err != nil { log.Warn("marshal metrics value failed", zap.Any("metrics", metrics), zap.String("err", err.Error())) @@ -45,3 +46,82 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) { } return string(bs), nil } + +func getSearchParamString(params []*commonpb.KeyValuePair) string { + searchParams := "" + for _, kv := range params { + searchParams += kv.Key + "=" + kv.Value + "," + } + if len(searchParams) > 0 { + searchParams = searchParams[:len(searchParams)-1] + } + return searchParams +} + +func NewSlowQueryWithQueryRequest(request *milvuspb.QueryRequest, user string, cost time.Duration, traceID string) *SlowQuery { + queryParams := &QueryParams{ + Expr: request.GetExpr(), + OutputFields: strings.Join(request.GetOutputFields(), ","), + } + + return &SlowQuery{ + Role: typeutil.ProxyRole, + Database: request.GetDbName(), + Collection: request.GetCollectionName(), + Partitions: strings.Join(request.GetPartitionNames(), ","), + ConsistencyLevel: request.GetConsistencyLevel().String(), + UseDefaultConsistency: request.GetUseDefaultConsistency(), + GuaranteeTimestamp: request.GetGuaranteeTimestamp(), + Duration: cost.String(), + User: user, + QueryParams: queryParams, + Type: "Query", + TraceID: traceID, + Time: time.Now().Format(time.DateTime), + } +} + +func NewSlowQueryWithSearchRequest(request *milvuspb.SearchRequest, user string, cost time.Duration, traceID string) *SlowQuery { + searchParams := getSearchParamString(request.GetSearchParams()) + + var subReqs []*SearchParams + for _, req := range request.GetSubReqs() { + subReqs = append(subReqs, &SearchParams{ + DSL: []string{req.GetDsl()}, + SearchParams: []string{getSearchParamString(req.GetSearchParams())}, + NQ: []int64{req.GetNq()}, + }) + } + + searchType := "HybridSearch" + if len(request.GetSubReqs()) == 0 { + subReqs = append(subReqs, &SearchParams{ + DSL: []string{request.GetDsl()}, + SearchParams: []string{searchParams}, + NQ: []int64{request.GetNq()}, + }) + searchType = "Search" + } + + queryParams := &QueryParams{ + SearchParams: subReqs, + Expr: request.GetDsl(), + OutputFields: strings.Join(request.GetOutputFields(), ","), + } + + return &SlowQuery{ + Role: typeutil.ProxyRole, + Database: request.GetDbName(), + Collection: request.GetCollectionName(), + Partitions: strings.Join(request.GetPartitionNames(), ","), + ConsistencyLevel: request.GetConsistencyLevel().String(), + UseDefaultConsistency: request.GetUseDefaultConsistency(), + GuaranteeTimestamp: request.GetGuaranteeTimestamp(), + Duration: cost.String(), + User: user, + QueryParams: queryParams, + Type: searchType, + TraceID: traceID, + Time: time.Now().Format(time.DateTime), + } +} diff --git a/pkg/util/metricsinfo/utils_test.go b/pkg/util/metricsinfo/utils_test.go index 0fa6561533..f3099014bf 100644 --- a/pkg/util/metricsinfo/utils_test.go +++ b/pkg/util/metricsinfo/utils_test.go @@ -13,8 +13,12 @@ package metricsinfo import ( "testing" + "time" "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" ) func TestFillDeployMetricsWithEnv(t *testing.T) { @@ -43,3 +47,74 @@ func TestFillDeployMetricsWithEnv(t *testing.T) { assert.Equal(t, goVersion, m.UsedGoVersion) assert.Equal(t, buildTime, m.BuildTime) } + +func TestNewSlowQueryWithSearchRequest(t *testing.T) { + request := &milvuspb.SearchRequest{ + DbName: "test_db", + CollectionName: "test_collection", + PartitionNames: []string{"partition1", "partition2"}, + ConsistencyLevel: commonpb.ConsistencyLevel_Bounded, + UseDefaultConsistency: true, + GuaranteeTimestamp: 123456789, + SearchParams: []*commonpb.KeyValuePair{{Key: "param1", Value: "value1"}}, + SubReqs: []*milvuspb.SubSearchRequest{{Dsl: "dsl1", SearchParams: []*commonpb.KeyValuePair{{Key: "param2", Value: "value2"}}, Nq: 10}}, + Dsl: "dsl2", + Nq: 20, + OutputFields: []string{"field1", "field2"}, + } + user := "test_user" + cost := time.Duration(100) * time.Millisecond + + slowQuery := NewSlowQueryWithSearchRequest(request, user, cost, "") + + assert.NotNil(t, slowQuery) + assert.Equal(t, "proxy", slowQuery.Role) + assert.Equal(t, "test_db", slowQuery.Database) + assert.Equal(t, "test_collection", slowQuery.Collection) + assert.Equal(t, "partition1,partition2", slowQuery.Partitions) + assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel) + assert.True(t, slowQuery.UseDefaultConsistency) + assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp) + assert.Equal(t, "100ms", slowQuery.Duration) + assert.Equal(t, user, slowQuery.User) + assert.Equal(t, "HybridSearch", slowQuery.Type) + assert.NotNil(t, slowQuery.QueryParams) + assert.Equal(t, "dsl2", slowQuery.QueryParams.Expr) + assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields) + assert.Len(t, slowQuery.QueryParams.SearchParams, 1) + assert.Equal(t, []string{"dsl1"}, slowQuery.QueryParams.SearchParams[0].DSL) + assert.Equal(t, []string{"param2=value2"}, slowQuery.QueryParams.SearchParams[0].SearchParams) + assert.Equal(t, []int64{10}, slowQuery.QueryParams.SearchParams[0].NQ) +} + +func TestNewSlowQueryWithQueryRequest(t *testing.T) { + request := &milvuspb.QueryRequest{ + DbName: "test_db", + CollectionName: "test_collection", + PartitionNames: []string{"partition1", "partition2"}, + ConsistencyLevel: commonpb.ConsistencyLevel_Bounded, + UseDefaultConsistency: true, + GuaranteeTimestamp: 123456789, + Expr: "expr1", + OutputFields: []string{"field1", "field2"}, + } + user := "test_user" + cost := time.Duration(100) * time.Millisecond + + slowQuery := NewSlowQueryWithQueryRequest(request, user, cost, "") + + assert.NotNil(t, slowQuery) + assert.Equal(t, "proxy", slowQuery.Role) + assert.Equal(t, "test_db", slowQuery.Database) + assert.Equal(t, "test_collection", slowQuery.Collection) + assert.Equal(t, "partition1,partition2", slowQuery.Partitions) + assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel) + assert.True(t, slowQuery.UseDefaultConsistency) + assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp) + assert.Equal(t, "100ms", slowQuery.Duration) + assert.Equal(t, user, slowQuery.User) + assert.Equal(t, "Query", slowQuery.Type) + assert.NotNil(t, slowQuery.QueryParams) + assert.Equal(t, "expr1", slowQuery.QueryParams.Expr) + assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields) +} diff --git a/tests/integration/minicluster_v2.go b/tests/integration/minicluster_v2.go index d6309a15d1..9b0f1d4452 100644 --- a/tests/integration/minicluster_v2.go +++ b/tests/integration/minicluster_v2.go @@ -464,14 +464,23 @@ func (cluster *MiniClusterV2) Stop() error { if cluster.clientConn != nil { cluster.clientConn.Close() } - cluster.RootCoord.Stop() - log.Info("mini cluster rootCoord stopped") - cluster.DataCoord.Stop() - log.Info("mini cluster dataCoord stopped") - cluster.QueryCoord.Stop() - log.Info("mini cluster queryCoord stopped") - cluster.Proxy.Stop() - log.Info("mini cluster proxy stopped") + if cluster.RootCoord != nil { + cluster.RootCoord.Stop() + log.Info("mini cluster rootCoord stopped") + } + + if cluster.DataCoord != nil { + cluster.DataCoord.Stop() + log.Info("mini cluster dataCoord stopped") + } + if cluster.QueryCoord != nil { + cluster.QueryCoord.Stop() + log.Info("mini cluster queryCoord stopped") + } + if cluster.Proxy != nil { + cluster.Proxy.Stop() + log.Info("mini cluster proxy stopped") + } cluster.StopAllDataNodes() cluster.StopAllStreamingNodes()