admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add segment/channel/task/slow query render (#37561)

Browse Source 
issue: #36621

Signed-off-by: jaime <yun.zhang@zilliz.com>
This commit is contained in:
jaime 2024-11-12 17:44:29 +08:00 committed by GitHub
parent 2742e9573f
commit 1e8ea4a7e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 2699 additions and 715 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/datacoord/compaction_task_meta.go
View File

@ -63,7 +63,7 @@ func newCompactionTaskMeta(ctx context.Context, catalog metastore.DataCoordCatal
ctx: ctx, ctx: ctx,
catalog: catalog, catalog: catalog,
compactionTasks: make(map[int64]map[int64]*datapb.CompactionTask, 0), compactionTasks: make(map[int64]map[int64]*datapb.CompactionTask, 0),
taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](1024, nil, time.Minute*60), taskStats: expirable.NewLRU[UniqueID, *metricsinfo.CompactionTask](32, nil, time.Minute*15),
} }
if err := csm.reloadFromKV(); err != nil { if err := csm.reloadFromKV(); err != nil {
return nil, err return nil, err
@ -178,10 +178,6 @@ func (csm *compactionTaskMeta) DropCompactionTask(task *datapb.CompactionTask) e
func (csm *compactionTaskMeta) TaskStatsJSON() string { func (csm *compactionTaskMeta) TaskStatsJSON() string {
tasks := csm.taskStats.Values() tasks := csm.taskStats.Values()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks) ret, err := json.Marshal(tasks)
if err != nil { if err != nil {
return "" return ""

2
internal/datacoord/compaction_task_meta_test.go
View File

@ -111,7 +111,7 @@ func (suite *CompactionTaskMetaSuite) TestTaskStatsJSON() {
// testing return empty string // testing return empty string
actualJSON := suite.meta.TaskStatsJSON() actualJSON := suite.meta.TaskStatsJSON()
suite.Equal("", actualJSON) suite.Equal("[]", actualJSON)
err := suite.meta.SaveCompactionTask(task1) err := suite.meta.SaveCompactionTask(task1)
suite.NoError(err) suite.NoError(err)

5
internal/datacoord/import_meta.go
View File

@ -52,7 +52,7 @@ type importTasks struct {
func newImportTasks() *importTasks { func newImportTasks() *importTasks {
return &importTasks{ return &importTasks{
tasks: make(map[int64]ImportTask), tasks: make(map[int64]ImportTask),
taskStats: expirable.NewLRU[UniqueID, ImportTask](4096, nil, time.Minute*60), taskStats: expirable.NewLRU[UniqueID, ImportTask](64, nil, time.Minute*30),
} }
} }
@ -301,9 +301,6 @@ func (m *importMeta) RemoveTask(taskID int64) error {
func (m *importMeta) TaskStatsJSON() string { func (m *importMeta) TaskStatsJSON() string {
tasks := m.tasks.listTaskStats() tasks := m.tasks.listTaskStats()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks) ret, err := json.Marshal(tasks)
if err != nil { if err != nil {

2
internal/datacoord/import_meta_test.go
View File

@ -251,7 +251,7 @@ func TestTaskStatsJSON(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
statsJSON := im.TaskStatsJSON() statsJSON := im.TaskStatsJSON()
assert.Equal(t, "", statsJSON) assert.Equal(t, "[]", statsJSON)
task1 := &importTask{ task1 := &importTask{
ImportTaskV2: &datapb.ImportTaskV2{ ImportTaskV2: &datapb.ImportTaskV2{

4
internal/datacoord/import_task.go
View File

@ -185,7 +185,7 @@ func (p *preImportTask) MarshalJSON() ([]byte, error) {
NodeID: p.GetNodeID(), NodeID: p.GetNodeID(),
State: p.GetState().String(), State: p.GetState().String(),
Reason: p.GetReason(), Reason: p.GetReason(),
TaskType: "PreImportTask", TaskType: p.GetType().String(),
CreatedTime: p.GetCreatedTime(), CreatedTime: p.GetCreatedTime(),
CompleteTime: p.GetCompleteTime(), CompleteTime: p.GetCompleteTime(),
} }
@ -231,7 +231,7 @@ func (t *importTask) MarshalJSON() ([]byte, error) {
NodeID: t.GetNodeID(), NodeID: t.GetNodeID(),
State: t.GetState().String(), State: t.GetState().String(),
Reason: t.GetReason(), Reason: t.GetReason(),
TaskType: "ImportTask", TaskType: t.GetType().String(),
CreatedTime: t.GetCreatedTime(), CreatedTime: t.GetCreatedTime(),
CompleteTime: t.GetCompleteTime(), CompleteTime: t.GetCompleteTime(),
} }

6
internal/datacoord/index_meta.go
View File

@ -102,7 +102,7 @@ func newSegmentIndexBuildInfo() *segmentBuildInfo {
// build ID -> segment index // build ID -> segment index
buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex), buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex),
// build ID -> task stats // build ID -> task stats
taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](1024, nil, time.Minute*60), taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](64, nil, time.Minute*30),
} }
} }
@ -1075,10 +1075,6 @@ func (m *indexMeta) HasIndex(collectionID int64) bool {
func (m *indexMeta) TaskStatsJSON() string { func (m *indexMeta) TaskStatsJSON() string {
tasks := m.segmentBuildInfo.GetTaskStats() tasks := m.segmentBuildInfo.GetTaskStats()
if len(tasks) == 0 {
return ""
}
ret, err := json.Marshal(tasks) ret, err := json.Marshal(tasks)
if err != nil { if err != nil {
return "" return ""

2
internal/datacoord/index_meta_test.go
View File

@ -1543,7 +1543,7 @@ func TestBuildIndexTaskStatsJSON(t *testing.T) {
} }
actualJSON := im.TaskStatsJSON() actualJSON := im.TaskStatsJSON()
assert.Equal(t, "", actualJSON) assert.Equal(t, "[]", actualJSON)
im.segmentBuildInfo.Add(si1) im.segmentBuildInfo.Add(si1)
im.segmentBuildInfo.Add(si2) im.segmentBuildInfo.Add(si2)

2
internal/datacoord/job_manager_test.go
View File

@ -104,7 +104,7 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() {
allocator: alloc, allocator: alloc,
tasks: make(map[int64]Task), tasks: make(map[int64]Task),
meta: mt, meta: mt,
taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5), taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*5),
}, },
allocator: alloc, allocator: alloc,
} }

6
internal/datacoord/metrics_info_test.go
View File

@ -325,7 +325,7 @@ func TestGetSyncTaskMetrics(t *testing.T) {
mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)}) mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)})
svr.cluster = mockCluster svr.cluster = mockCluster
expectedJSON := "" expectedJSON := "null"
actualJSON, err := svr.getSyncTaskJSON(ctx, req) actualJSON, err := svr.getSyncTaskJSON(ctx, req)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON) assert.Equal(t, expectedJSON, actualJSON)
@ -449,7 +449,7 @@ func TestGetSegmentsJSON(t *testing.T) {
mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)}) mockCluster.EXPECT().GetSessions().Return([]*session.Session{session.NewSession(&session.NodeInfo{NodeID: 1}, dataNodeCreator)})
svr.cluster = mockCluster svr.cluster = mockCluster
expectedJSON := "" expectedJSON := "null"
actualJSON, err := svr.getSegmentsJSON(ctx, req) actualJSON, err := svr.getSegmentsJSON(ctx, req)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON) assert.Equal(t, expectedJSON, actualJSON)
@ -591,7 +591,7 @@ func TestGetChannelsJSON(t *testing.T) {
svr.cluster = mockCluster svr.cluster = mockCluster
svr.meta = &meta{channelCPs: newChannelCps()} svr.meta = &meta{channelCPs: newChannelCps()}
expectedJSON := "" expectedJSON := "null"
actualJSON, err := svr.getChannelsJSON(ctx, req) actualJSON, err := svr.getChannelsJSON(ctx, req)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, expectedJSON, actualJSON) assert.Equal(t, expectedJSON, actualJSON)

2
internal/datacoord/task_scheduler.go
View File

@ -91,7 +91,7 @@ func newTaskScheduler(
handler: handler, handler: handler,
indexEngineVersionManager: indexEngineVersionManager, indexEngineVersionManager: indexEngineVersionManager,
allocator: allocator, allocator: allocator,
taskStats: expirable.NewLRU[UniqueID, Task](1024, nil, time.Minute*5), taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15),
} }
ts.reloadFromMeta() ts.reloadFromMeta()
return ts return ts

21
internal/distributed/proxy/httpserver/handler.go
View File

@ -27,6 +27,9 @@ func (h *Handlers) RegisterRoutesTo(router gin.IRouter) {
router.GET("/health", wrapHandler(h.handleGetHealth)) router.GET("/health", wrapHandler(h.handleGetHealth))
router.POST("/dummy", wrapHandler(h.handleDummy)) router.POST("/dummy", wrapHandler(h.handleDummy))
router.GET("/databases", wrapHandler(h.handleListDatabases))
router.GET("/database", wrapHandler(h.handleDescribeDatabases))
router.POST("/collection", wrapHandler(h.handleCreateCollection)) router.POST("/collection", wrapHandler(h.handleCreateCollection))
router.DELETE("/collection", wrapHandler(h.handleDropCollection)) router.DELETE("/collection", wrapHandler(h.handleDropCollection))
router.GET("/collection/existence", wrapHandler(h.handleHasCollection)) router.GET("/collection/existence", wrapHandler(h.handleHasCollection))
@ -96,6 +99,24 @@ func (h *Handlers) handleDummy(c *gin.Context) (interface{}, error) {
return h.proxy.Dummy(c, &req) return h.proxy.Dummy(c, &req)
} }
func (h *Handlers) handleListDatabases(c *gin.Context) (interface{}, error) {
req := milvuspb.ListDatabasesRequest{}
err := shouldBind(c, &req)
if err != nil {
return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err)
}
return h.proxy.ListDatabases(c, &req)
}
func (h *Handlers) handleDescribeDatabases(c *gin.Context) (interface{}, error) {
req := milvuspb.DescribeDatabaseRequest{}
err := shouldBind(c, &req)
if err != nil {
return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err)
}
return h.proxy.DescribeDatabase(c, &req)
}
func (h *Handlers) handleCreateCollection(c *gin.Context) (interface{}, error) { func (h *Handlers) handleCreateCollection(c *gin.Context) (interface{}, error) {
wrappedReq := WrappedCreateCollectionRequest{} wrappedReq := WrappedCreateCollectionRequest{}
err := shouldBind(c, &wrappedReq) err := shouldBind(c, &wrappedReq)

2
internal/flushcommon/syncmgr/sync_manager.go
View File

@ -70,7 +70,7 @@ func NewSyncManager(chunkManager storage.ChunkManager) SyncManager {
keyLockDispatcher: dispatcher, keyLockDispatcher: dispatcher,
chunkManager: chunkManager, chunkManager: chunkManager,
tasks: typeutil.NewConcurrentMap[string, Task](), tasks: typeutil.NewConcurrentMap[string, Task](),
taskStats: expirable.NewLRU[string, Task](512, nil, time.Minute*15), taskStats: expirable.NewLRU[string, Task](16, nil, time.Minute*15),
} }
// setup config update watcher // setup config update watcher
params.Watch(params.DataNodeCfg.MaxParallelSyncMgrTasks.Key, config.NewHandler("datanode.syncmgr.poolsize", syncMgr.resizeHandler)) params.Watch(params.DataNodeCfg.MaxParallelSyncMgrTasks.Key, config.NewHandler("datanode.syncmgr.poolsize", syncMgr.resizeHandler))

2
internal/http/router.go
View File

@ -75,6 +75,8 @@ const (
ClusterDependenciesPath = "/_cluster/dependencies" ClusterDependenciesPath = "/_cluster/dependencies"
// HookConfigsPath is the path to get hook configurations. // HookConfigsPath is the path to get hook configurations.
HookConfigsPath = "/_hook/configs" HookConfigsPath = "/_hook/configs"
// SlowQueryPath is the path to get slow queries metrics
SlowQueryPath = "/_cluster/slow_query"
// QCDistPath is the path to get QueryCoord distribution. // QCDistPath is the path to get QueryCoord distribution.
QCDistPath = "/_qc/dist" QCDistPath = "/_qc/dist"

158
internal/http/webui/channels.html
View File

@ -1,158 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Channels</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>
Channel Checkpoints
</h2>
<table id="channelCP" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Checkpoint Ts</th>
<th scope="col">Checkpoint Offset</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>11</td>
<td>2022-11-11 12:00:00</td>
<td>{ledgerID:1, entryID:1, batchIdx:0}</td>
<td>datanode1</td>
</tr>
<tr>
<td>channel2</td>
<td>22</td>
<td>2022-11-11 12:00:00</td>
<td>{ledgerID:2, entryID:2, batchIdx:0}</td>
<td>datanode1</td>
</tr>
</tbody>
</table>
<h2>
Watched Channels On Datanode
</h2>
<table id="watchedChDataNode" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Consume Rate/s</th>
<th scope="col">Latency</th>
<th scope="col">TimeTickLag</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>datanode1</td>
</tr>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watched</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<h2>
Watched Channels On QueryNode
</h2>
<table id="watchedChQueryNode" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Consume Rate/s</th>
<!-- consume latency-->
<th scope="col">Latency</th>
<th scope="col">TimeTickLag</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>channel1</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>querynode1</td>
</tr>
<tr>
<td>channel2</td>
<td>200</td>
<td>11</td>
<td>50ms</td>
<td>100ms</td>
<td>watching</td>
<td>querynode2</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
// load cluster information data
document.addEventListener("DOMContentLoaded", function() {
fetch(MILVUS_URI + "/")
.then(text => {
//TODO add channel render
throw new Error("Unimplemented API")
})
.catch(error => {
handleError(error);
});
});
</script>
</body>
</html>

157
internal/http/webui/collections.html
View File

@ -23,111 +23,53 @@
</div> </div>
<div class="col-md-8"> <div class="col-md-8">
<h2> <h2>
Database List Database
</h2> </h2>
<table id="database" class="table table-hover"> <table id="database" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Database ID</th>
<th scope="col">Name</th>
<th scope="col">Create Time</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>db1</td>
<td>2022-11-11 12:00:00</td>
</tr>
<tr>
<td>2</td>
<td>db2</td>
<td>2022-11-11 12:00:00</td>
</tr>
</tbody>
</table> </table>
<div style="display: flex; justify-content: space-between;">
<p id="db_totalCount"></p>
<p id="dbPaginationControls"></p>
</div>
<h2> <h2>
Collection List Collection
</h2> </h2>
<table id="collectionMeta" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Collection ID</th>
<th scope="col">Collection Name</th>
<th scope="col">Partition Key Count</th>
<th scope="col">Partition Count</th>
<th scope="col">Channel Count</th>
<th scope="col">Segment Count</th>
<th scope="col">Binlog Count</th>
<th scope="col">Size</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>db1.coll1-fake</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>1</td>
</tr>
<tr>
<td>2</td>
<td>db1.coll2-fake</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
<td>2</td>
</tr>
</tbody>
</table>
<!-- Navigation Tabs -->
<ul class="nav nav-tabs" id="componentTabs" role="tablist">
<li class="nav-item">
<a class="nav-link active" id="coll-base-stats-tab" data-toggle="tab" href="#coll-base-stats" role="tab" aria-controls="coll-base-stats" aria-selected="true">Base </a>
</li>
<li class="nav-item">
<a class="nav-link" id="coll-request-tab" data-toggle="tab" href="#coll-request-metrics" role="tab" aria-controls="coll-request-metrics" aria-selected="false">Requests</a>
</li>
<h2> <!-- Search Input -->
Collection Metrics <li class="nav-item ml-auto">
</h2> <input type="text" class="form-control" placeholder="Search database..." id="databaseSearch" oninput="searchCollections()">
<table id="collectionMetrics" class="table table-hover"> </li>
<thead class="thead-light"> </ul>
<tr>
<th scope="col">Collection Name</th> <!-- Tab Content -->
<th scope="col">isQueryable</th> <div class="tab-content" id="componentTabsContent">
<th scope="col">isWritable</th> <div class="tab-pane fade show active" id="coll-base-stats" role="tabpanel" aria-labelledby="coll-base-stats-tab">
<th scope="col">Query Ops/s</th> <table id="collectionList" class="table table-hover"></table>
<th scope="col">Search Ops/s</th>
<th scope="col">Insert Throughput(MB/s)</th>
<th scope="col">Delete Throughput(MB/s)</th>
</tr>
</thead>
<tbody>
<tr>
<td>db1.coll-fake</td>
<td>true</td>
<td>true</td>
<td>20</td>
<td>20</td>
<td>1</td>
<td>0.5</td>
</tr>
<tr>
<td>db1.col2-fake</td>
<td>true</td>
<td>true</td>
<td>20</td>
<td>20</td>
<td>1</td>
<td>0.5</td>
</tr>
</tbody>
</table>
</div> </div>
<div class="col-md-2">
<div class="tab-pane fade" id="coll-request-metrics" role="tabpanel" aria-labelledby="coll-request-tab">
<table id="collectionRequests" class="table table-hover mt-3"></table>
</div> </div>
</div> </div>
<div style="display: flex; justify-content: space-between;">
<p id="collection_totalCount"></p>
<p id="collectionPaginationControls"></p>
</div>
</div>
<div class="col-md-2"></div>
</div>
<div id="footer"></div> <div id="footer"></div>
</div> </div>
@ -137,12 +79,33 @@
$('#footer').load("footer.html"); $('#footer').load("footer.html");
}); });
fetchData(MILVUS_URI + "/", clientInfos) function searchCollections() {
const searchTerm = document.getElementById('databaseSearch').value;
let dbName = 'default';
if (searchTerm !== '') {
dbName = searchTerm;
}
fetchCollections(dbName);
}
searchCollections()
// TODO - Implement the following functions and support search with db name
// fetchData(MILVUS_URI + "/_collection/metrics", collectionRequest)
// .then(data => {
// collectionRequestsData = data;
// renderCollectionRequests(startPage, paginationSize);
// })
// .catch(error => {
// handleError(error);
// });
fetchData(MILVUS_URI + "/databases", databases)
.then(data => { .then(data => {
//TODO add collection render databaseData = data;
renderDatabases(startPage, paginationSize)
}) })
.catch(error => { .catch(error => {
handleError(new Error("Unimplemented API")); handleError(error);
}); });
</script> </script>
</body> </body>

11
internal/http/webui/configs.html
View File

@ -24,7 +24,12 @@
<h2> <h2>
Milvus Configurations Milvus Configurations
</h2> </h2>
<div>
<input type="text" id="searchInput" placeholder="Search..." oninput="searchConfigs()" class="form-control mb-3" />
</div>
<table id="mConfig" class="table table-hover"></table> <table id="mConfig" class="table table-hover"></table>
<div id="paginationControls"></div>
<h2> <h2>
Hook Configurations Hook Configurations
@ -41,6 +46,12 @@
$('#footer').load("footer.html"); $('#footer').load("footer.html");
}); });
function searchConfigs() {
const searchTerm = document.getElementById('searchInput').value;
currentPage = 0; // Reset to the first page on new search
renderConfigs(configData, searchTerm);
}
// load cluster configurations // load cluster configurations
fetchData(MILVUS_URI + '/_cluster/configs', mconfigs) fetchData(MILVUS_URI + '/_cluster/configs', mconfigs)
.then(data => { .then(data => {

90
internal/http/webui/data_component.html Normal file
View File

@ -0,0 +1,90 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Data Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>Data Channels</h2>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Watch State</th>
<th scope="col">Node ID</th>
<th scope="col">Latest Time Tick</th>
<th scope="col">Start Watch Ts</th>
<th scope="col">Checkpoint Ts</th>
</tr>
</thead>
<tbody id="dataChannelsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<p id="dchannel_totalCount"></p>
<div id="dchannelPaginationControls"></div>
</div>
<!-- Notifications Container -->
<div id="notificationsDChannels"></div>
<h2>Data Segments</h2>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Segment ID</th>
<th scope="col">Collection ID</th>
<th scope="col">Partition ID</th>
<th scope="col">Channel</th>
<th scope="col">Num of Rows</th>
<th scope="col">State</th>
<th scope="col">Level</th>
</tr>
</thead>
<tbody id="dataSegmentsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<p id="dsegment_totalCount"></p>
<div id="dsegmentPaginationControls"></div>
</div>
<!-- Notifications Container -->
<div id="notificationsDSegments"></div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
fetchAndRenderDataChannels(startPage, paginationSize);
fetchAndRenderDataSegments(dataSegmentsStartPage, dataSegmentsPaginationSize);
</script>
</body>
</html>

10
internal/http/webui/header.html
View File

@ -11,17 +11,23 @@
<a class="nav-link" href="collections.html">Collections</a> <a class="nav-link" href="collections.html">Collections</a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="segments.html">Segments</a> <a class="nav-link" href="query_component.html">Query</a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="channels.html">Channels</a> <a class="nav-link" href="data_component.html">Data</a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="tasks.html">Tasks</a> <a class="nav-link" href="tasks.html">Tasks</a>
</li> </li>
<li class="nav-item">
<a class="nav-link " href="slow_requests.html">Slow Requests</a>
</li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link " href="configs.html">Configurations</a> <a class="nav-link " href="configs.html">Configurations</a>
</li> </li>
<li class="nav-item">
<a class="nav-link " href="tools.html">Tools</a>
</li>
</ul> </ul>
</div> </div>
</nav> </nav>

16
internal/http/webui/index.html
View File

@ -39,6 +39,9 @@
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" id="runtime-metrics-tab" data-toggle="tab" href="#runtime-metrics" role="tab" aria-controls="runtime-metrics" aria-selected="false">Runtime Metrics</a> <a class="nav-link" id="runtime-metrics-tab" data-toggle="tab" href="#runtime-metrics" role="tab" aria-controls="runtime-metrics" aria-selected="false">Runtime Metrics</a>
</li> </li>
<li class="nav-item">
<a class="nav-link" id="request-tab" data-toggle="tab" href="#request-metrics" role="tab" aria-controls="request-metrics" aria-selected="false">Requests</a>
</li>
</ul> </ul>
<!-- Tab Content --> <!-- Tab Content -->
@ -52,6 +55,11 @@
<div class="tab-pane fade" id="runtime-metrics" role="tabpanel" aria-labelledby="runtime-metrics-tab"> <div class="tab-pane fade" id="runtime-metrics" role="tabpanel" aria-labelledby="runtime-metrics-tab">
<table id="nodeMetrics" class="table table-hover mt-3"></table> <table id="nodeMetrics" class="table table-hover mt-3"></table>
</div> </div>
<!-- Node Request Table -->
<div class="tab-pane fade" id="request-metrics" role="tabpanel" aria-labelledby="request-tab">
<table id="nodeRequests" class="table table-hover mt-3"></table>
</div>
</div> </div>
<h2> <h2>
@ -86,6 +94,14 @@
handleError(error); handleError(error);
}); });
// fetchData(MILVUS_URI + "/_node/requests", nodeRequests)
// .then(data => {
// renderNodeRequests(data)
// })
// .catch(error => {
// handleError(error);
// });
fetchData(MILVUS_URI + "/_cluster/clients", clientInfos) fetchData(MILVUS_URI + "/_cluster/clients", clientInfos)
.then(data => { .then(data => {
renderClientsInfo(data); renderClientsInfo(data);

63
internal/http/webui/query_component.html Normal file
View File

@ -0,0 +1,63 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Query Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h3>Segments</h3>
<table id="querySegmentsTable" class="table table-bordered"></table>
<div style="display: flex; justify-content: space-between;">
<div id="querySegmentsTotalCount"></div>
<div id="querySegmentsPagination"></div>
</div>
<h3>Channels</h3>
<table id="queryChannelsTable" class="table table-bordered"></table>
<div style="display: flex; justify-content: space-between;">
<div id="queryChannelsTotalCount"></div>
<div id="queryChannelsPagination"></div>
</div>
<h3>Replicas</h3>
<table id="replicasTable" class="table table-bordered"></table>
<h3>Resource Groups</h3>
<table id="resourceGroupTable" class="table table-bordered"></table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
fetchAndRenderQuerySegmentsAndChannels();
fetchAndRenderReplicas();
fetchAndRenderResourceGroup()
</script>
</body>
</html>

86
internal/http/webui/query_target.html Normal file
View File

@ -0,0 +1,86 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Query Component</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h3>Target Segments</h3>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Segment ID</th>
<th scope="col">Collection ID</th>
<th scope="col">Partition ID</th>
<th scope="col">Channel</th>
<th scope="col">Num of Rows</th>
<th scope="col">State</th>
<th scope="col">Target Scope</th>
</tr>
</thead>
<tbody id="dataSegmentsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<div id="segment_totalCount"></div>
<div id="segmentPaginationControls"></div>
</div>
<h3>Target Channels</h3>
<table class="table table-bordered">
<thead class="thead-light">
<tr>
<th scope="col">Channel Name</th>
<th scope="col">Collection ID</th>
<th scope="col">Node ID</th>
<th scope="col">Version</th>
<th scope="col">Unflushed Segments</th>
<th scope="col">Flushed Segments</th>
<th scope="col">Dropped Segments</th>
<th scope="col">Target Scope</th>
</tr>
</thead>
<tbody id="dataChannelsTableBody"></tbody>
</table>
<div style="display: flex; justify-content: space-between;">
<div id="channel_totalCount"></div>
<div id="channelPaginationControls"></div>
</div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
let targetTableStartPage = 0
let pageSize = 10
fetchAndRenderTargets(targetTableStartPage, pageSize)
</script>
</body>
</html>

196
internal/http/webui/segments.html
View File

@ -1,196 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Milvus WebUI - Segments</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<h2>
Loading Segments
</h2>
<!-- TODO: -->
<table id="loadingSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
Releasing Segments
</h2>
<!-- TODO: -->
<table id="releasingSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
Loaded Segments
</h2>
<!-- TODO: -->
<table id="loadedSegments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">isIndexed</th>
<th scope="col">segmentSize</th>
<th scope="col">queryNode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>false</td>
<td>6</td>
<td>faked-querynode1</td>
</tr>
<tr>
<td>111</td>
<td>coll1</td>
<td>11</td>
<td>6</td>
<td>faked-querynode2</td>
</tr>
</tbody>
</table>
<h2>
All Segments
</h2>
<!-- TODO: data from data component -->
<table id="segments" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">segmentID</th>
<th scope="col">collectionName</th>
<th scope="col">state</th>
<th scope="col">rowCount</th>
<th scope="col">binlog count</th>
<th scope="col">binlogs size</th>
<th scope="col">statslogs size</th>
<th scope="col">dmlChannel</th>
<th scope="col">level</th>
<th scope="col">datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>111</td>
<td>coll1</td>
<td>Flushing</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>faked-channel-1</td>
<td>L0</td>
<td>faked-datanode1</td>
</tr>
<tr>
<td>22222</td>
<td>coll1</td>
<td>Flushing</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>11</td>
<td>faked-channel-2</td>
<td>L0</td>
<td>faked-datanode2</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/", sysmetrics)
.then(data => {
//TODO add segment render
})
.catch(error => {
handleError(new Error("Unimplemented API"));
});
});
</script>
</body>
</html>

58
internal/http/webui/slow_requests.html Normal file
View File

@ -0,0 +1,58 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Slow Requests</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<div class="alert alert-warning alert-dismissible fade show" role="alert">
<strong>Notice:</strong> Slow request in the last 5 minutes.
<!-- <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>-->
</div>
<table id="slowQueries" class="table table-bordered table-hover table-auto">
</table>
<p id="slowQueriesTotalCount"></p>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/_cluster/slow_query", slowQueries)
.then(data => {
renderSlowQueries(data);
})
.catch(error => {
handleError(error);
});
});
</script>
</body>
</html>

6
internal/http/webui/static/css/style.css
View File

@ -32,3 +32,9 @@
background-color: #000; background-color: #000;
margin: 40px 0; margin: 40px 0;
} }
.table td.fit,
.table th.fit {
white-space: nowrap;
width: 1%;
}

35
internal/http/webui/static/js/common.js
View File

@ -27,18 +27,29 @@ toggleDebugMode();
const handleError = (error) => { const handleError = (error) => {
console.error('Error fetching data:', error); console.error('Error fetching data:', error);
const errorMessage = encodeURIComponent(error.message || 'Unknown error'); // const errorMessage = encodeURIComponent(error.message || 'Unknown error');
window.location.href = `5xx.html?error=${errorMessage}`; // window.location.href = `5xx.html?error=${errorMessage}`;
}; };
const fetchData = (url, localData) => { const fetchData = (url, localData, kvParams) => {
if (DEBUG_MODE) { if (DEBUG_MODE) {
return new Promise((resolve) => { return new Promise((resolve) => {
resolve(JSON.parse(localData)); resolve(JSON.parse(localData));
}); });
} else if (kvParams && kvParams.length !== 0) {
return fetch(url, {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json',
},
mode: 'no-cors',
body: JSON.stringify(kvParams)
}).then(response => response.json())
} else { } else {
return fetch(url) return fetch(url).then(response => {
.then(response => response.json()) return response.json();
})
} }
}; };
@ -52,3 +63,17 @@ function getQueryParams() {
}); });
return params; return params;
} }
function formatTimestamp(timestamp) {
const date = new Date(timestamp); // Convert timestamp to a Date object
// Format the date components
const year = date.getFullYear();
const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are zero-indexed
const day = ('0' + date.getDate()).slice(-2);
const hours = ('0' + date.getHours()).slice(-2);
const minutes = ('0' + date.getMinutes()).slice(-2);
const seconds = ('0' + date.getSeconds()).slice(-2);
// Return formatted date string
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`
}

567
internal/http/webui/static/js/mockdata.js
View File

@ -1,4 +1,4 @@
var sysmetrics = `{ const sysmetrics = `{
"nodes_info": [ "nodes_info": [
{ {
"identifier": 1, "identifier": 1,
@ -341,7 +341,40 @@ var sysmetrics = `{
] ]
}` }`
var clientInfos = `[ const nodeRequests = `
[
{
"node_name": "querynode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "datanode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "indexnode1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
},
{
"node_name": "proxy1",
"QPS": 0,
"read_request_count": 0,
"write_request_count": 0,
"delete_request_count": 0
}
]
`
const clientInfos = `[
{ {
"sdk_type": "python", "sdk_type": "python",
"sdk_version": "1.0.0", "sdk_version": "1.0.0",
@ -364,7 +397,7 @@ var clientInfos = `[
} }
]` ]`
var dependencies = ` const dependencies = `
{ {
"metastore": { "metastore": {
"health_status": true, "health_status": true,
@ -386,7 +419,7 @@ var dependencies = `
} }
` `
var mconfigs = ` const mconfigs = `
{ {
"MILVUS_GIT_BUILD_TAGS": "v2.2-testing-20240702-811-g38211f2b81-dev", "MILVUS_GIT_BUILD_TAGS": "v2.2-testing-20240702-811-g38211f2b81-dev",
"MILVUS_GIT_COMMIT": "38211f2b81", "MILVUS_GIT_COMMIT": "38211f2b81",
@ -412,7 +445,223 @@ var mconfigs = `
} }
`; `;
var qcTargets = ` const collections =`
{
"status": {
"error_code": "Success",
"reason": ""
},
"collection_names": [
"collection1",
"collection2",
"collection3",
"collection4",
"collection5",
"collection6",
"collection7",
"collection8",
"collection9",
"collection10"
],
"collection_ids": [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
],
"created_timestamps": [
1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400
],
"created_utc_timestamps": [
1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400
],
"inMemory_percentages": [
100, 90, 80, 70, 60, 50, 40, 30, 20, 10
],
"query_service_available": [
true, false, false, false, false, false, false, false, false, false
]
}
`
const collectionRequest = `
[
{
"collection_name": "collection1",
"search_QPS": 10,
"query_QPS": 5,
"write_throughput": 20,
"delete_QPS": 2
},
{
"collection_name": "collection2",
"search_QPS": 15,
"query_QPS": 7,
"write_throughput": 25,
"delete_QPS": 3
},
{
"collection_name": "collection3",
"search_QPS": 20,
"query_QPS": 10,
"write_throughput": 30,
"delete_QPS": 4
},
{
"collection_name": "collection4",
"search_QPS": 25,
"query_QPS": 12,
"write_throughput": 35,
"delete_QPS": 5
},
{
"collection_name": "collection5",
"search_QPS": 30,
"query_QPS": 15,
"write_throughput": 40,
"delete_QPS": 6
},
{
"collection_name": "collection6",
"search_QPS": 35,
"query_QPS": 17,
"write_throughput": 45,
"delete_QPS": 7
},
{
"collection_name": "collection7",
"search_QPS": 40,
"query_QPS": 20,
"write_throughput": 50,
"delete_QPS": 8
},
{
"collection_name": "collection8",
"search_QPS": 45,
"query_QPS": 22,
"write_throughput": 55,
"delete_QPS": 9
},
{
"collection_name": "collection9",
"search_QPS": 50,
"query_QPS": 25,
"write_throughput": 60,
"delete_QPS": 10
},
{
"collection_name": "collection10",
"search_QPS": 55,
"query_QPS": 27,
"write_throughput": 65,
"delete_QPS": 11
}
]
`
const describeCollectionResp = `
{
"status": {
"error_code": 0,
"reason": "Success"
},
"schema": {
"name": "example_collection",
"description": "This is an example collection schema",
"fields": [
{
"name": "field1",
"data_type": "INT64",
"is_primary_key": true,
"auto_id": false
},
{
"name": "field2",
"data_type": "FLOAT",
"is_primary_key": false,
"auto_id": false
}
]
},
"collectionID": 12345,
"virtual_channel_names": ["vchan1", "vchan2"],
"physical_channel_names": ["pchan1", "pchan2"],
"created_timestamp": 1633036800,
"created_utc_timestamp": 1633036800,
"shards_num": 2,
"aliases": ["alias1", "alias2"],
"start_positions": [
{
"key": "start_key",
"data": "start_data"
}
],
"consistency_level": 0,
"collection_name": "example_collection",
"properties": [
{
"key": "property_key",
"value": "property_value"
}
],
"db_name": "example_db",
"num_partitions": 1,
"db_id": 1
}
`
const databases = `
{
"status": {
"error_code": "Success",
"reason": ""
},
"db_names": [
"database_1",
"database_2",
"database_3",
"database_4",
"database_5",
"database_6",
"database_7",
"database_8",
"database_9",
"database_10"
],
"created_timestamp": [
1633036800,
1633123200,
1633209600,
1633296000,
1633382400,
1633468800,
1633555200,
1633641600,
1633728000,
1633814400
],
"db_ids": [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
]
}
`
const describeDatabaseResp = `
{
"status": {
"error_code": 0,
"reason": "Success"
},
"db_name": "example_db",
"dbID": 1,
"created_timestamp": 1633036800,
"properties": [
{
"key": "property_key",
"value": "property_value"
}
]
}
`
const qcCurrentTargets = `
[ [
{ {
"collection_id": 1, "collection_id": 1,
@ -442,7 +691,7 @@ var qcTargets = `
], ],
"resource_group": "rg1", "resource_group": "rg1",
"loaded_insert_row_count": 1000, "loaded_insert_row_count": 1000,
"mem_size": 2048, "mem_size": 2048
} }
], ],
"dm_channels": [ "dm_channels": [
@ -451,10 +700,18 @@ var qcTargets = `
"version": 1, "version": 1,
"collection_id": 1, "collection_id": 1,
"channel_name": "channel1", "channel_name": "channel1",
"unflushed_segment_ids": [1], "unflushed_segment_ids": [
"flushed_segment_ids": [2], 1
"dropped_segment_ids": [3], ],
"level_zero_segment_ids": [4], "flushed_segment_ids": [
2
],
"dropped_segment_ids": [
3
],
"level_zero_segment_ids": [
4
],
"partition_stats_versions": { "partition_stats_versions": {
"1": 1 "1": 1
} }
@ -464,7 +721,44 @@ var qcTargets = `
] ]
` `
var qcDist =` const qcNextTargets = `
[
{
"collection_id": 1,
"segments": [
{
"segment_id": 2,
"collection_id": 1,
"partition_id": 1,
"channel": "channel2",
"num_of_rows": 1000,
"state": "Sealed",
"is_importing": false,
"compacted": false,
"level": "L0",
"is_sorted": true,
"node_id": 1,
"is_invisible": false,
"loaded_timestamp": 1633072800,
"index": [
{
"field_id": 1,
"index_id": 1,
"build_id": 1,
"index_size": 1024,
"is_loaded": true
}
],
"resource_group": "rg1",
"loaded_insert_row_count": 1000,
"mem_size": 2048
}
]
}
]
`;
const qcDist = `
{ {
"segments": [ "segments": [
{ {
@ -493,6 +787,9 @@ var qcDist =`
"resource_group": "rg1", "resource_group": "rg1",
"loaded_insert_row_count": 1000, "loaded_insert_row_count": 1000,
"mem_size": 2048, "mem_size": 2048,
"flushed_rows": 1000,
"sync_buffer_rows": 0,
"syncing_rows": 0
} }
], ],
"dm_channels": [ "dm_channels": [
@ -507,18 +804,24 @@ var qcDist =`
"level_zero_segment_ids": [4], "level_zero_segment_ids": [4],
"partition_stats_versions": { "partition_stats_versions": {
"1": 1 "1": 1
} },
"watch_state": "Healthy",
"start_watch_ts": 1633072800
} }
], ],
"leader_views": [ "leader_views": [
{ {
"node_id": 1, "leader_id": 1,
"collection_id": 1, "collection_id": 1,
"channel_name": "channel1", "node_id": 1,
"segments": [ "channel": "channel1",
"version": 1,
"sealed_segments": [
{ {
"segment_id": 1, "segment_id": 1,
"collection_id": 1,
"partition_id": 1, "partition_id": 1,
"channel": "channel1",
"num_of_rows": 1000, "num_of_rows": 1000,
"state": "Sealed", "state": "Sealed",
"is_importing": false, "is_importing": false,
@ -540,52 +843,63 @@ var qcDist =`
"resource_group": "rg1", "resource_group": "rg1",
"loaded_insert_row_count": 1000, "loaded_insert_row_count": 1000,
"mem_size": 2048, "mem_size": 2048,
"flushed_rows": 1000,
"sync_buffer_rows": 0,
"syncing_rows": 0
} }
] ],
"growing_segments": [],
"target_version": 1,
"num_of_growing_rows": 0,
"unserviceable_error": ""
} }
] ]
} }
` `
var qcReplica = ` const qcReplica = `
[ [
{ {
"ID": 1, "ID": 1,
"CollectionID": 1, "CollectionID": 1,
"RWNodes": [1, 2], "rw_nodes": [1, 2],
"ResourceGroup": "rg1", "resource_group": "rg1",
"RONodes": [3], "ro_nodes": [3],
"ChannelToRWNodes": { "channel_to_rw_nodes": {
"channel1": [1, 2] "channel1": [1, 2]
} }
}, },
{ {
"ID": 2, "ID": 2,
"CollectionID": 2, "CollectionID": 2,
"RWNodes": [4, 5], "rw_nodes": [4, 5],
"ResourceGroup": "rg2", "resource_group": "rg2",
"RONodes": [6], "ro_nodes": [6],
"ChannelToRWNodes": { "channel_to_rw_nodes": {
"channel2": [4, 5] "channel2": [4, 5]
} }
} }
] ]
` `;
var qcResourceGroup = ` const qcResourceGroup = `
[ [
{ {
"Name": "rg1", "name": "rg1",
"Nodes": [1, 2] "nodes": [1, 2],
"cfg": {
"requests":{},
"limits":{"node_num":1000000}
}
}, },
{ {
"Name": "rg2", "name": "rg2",
"Nodes": [3, 4] "nodes": [3, 4]
} }
] ]
` `;
var qcTasks = ` const qcTasks = `
[ [
{ {
"task_name": "balance_checker-ChannelTask[1]-ch1", "task_name": "balance_checker-ChannelTask[1]-ch1",
@ -598,7 +912,7 @@ var qcTasks = `
"type:Grow node id : 1 channel name:channel_1" "type:Grow node id : 1 channel name:channel_1"
], ],
"step": 1, "step": 1,
"reason": "some reason" "reason": ""
}, },
{ {
"task_name": "index_checker-SegmentTask[2]-54321", "task_name": "index_checker-SegmentTask[2]-54321",
@ -611,7 +925,7 @@ var qcTasks = `
"type:Grow node id: 2 segment id:123 scope:DataScope_Streaming" "type:Grow node id: 2 segment id:123 scope:DataScope_Streaming"
], ],
"step": 2, "step": 2,
"reason": "another reason" "reason": ""
}, },
{ {
"task_name": "leader_checker-LeaderSegmentTask[3]-1", "task_name": "leader_checker-LeaderSegmentTask[3]-1",
@ -627,9 +941,9 @@ var qcTasks = `
"reason": "yet another reason" "reason": "yet another reason"
} }
] ]
` `;
var qn_segments = ` const qnSegments = `
[ [
{ {
"segment_id": 1, "segment_id": 1,
@ -656,7 +970,7 @@ var qn_segments = `
], ],
"resource_group": "rg1", "resource_group": "rg1",
"loaded_insert_row_count": 1000, "loaded_insert_row_count": 1000,
"mem_size": 2048, "mem_size": 2048
}, },
{ {
"segment_id": 2, "segment_id": 2,
@ -683,12 +997,12 @@ var qn_segments = `
], ],
"resource_group": "rg2", "resource_group": "rg2",
"loaded_insert_row_count": 2000, "loaded_insert_row_count": 2000,
"mem_size": 4096, "mem_size": 4096
} }
] ]
` `;
var qn_channels = ` const qnChannels = `
[ [
{ {
"name": "channel1", "name": "channel1",
@ -696,7 +1010,7 @@ var qn_channels = `
"assign_state": "assigned", "assign_state": "assigned",
"latest_time_tick": "2023-10-01 12:00:00", "latest_time_tick": "2023-10-01 12:00:00",
"node_id": 1, "node_id": 1,
"collection_id": 1, "collection_id": 1
}, },
{ {
"name": "channel2", "name": "channel2",
@ -704,26 +1018,39 @@ var qn_channels = `
"assign_state": "assigned", "assign_state": "assigned",
"latest_time_tick": "2023-10-01 12:05:00", "latest_time_tick": "2023-10-01 12:05:00",
"node_id": 2, "node_id": 2,
"collection_id": 2, "collection_id": 2
} }
] ]
` `;
var dc_dist = ` const dc_dist = `
{ {
"segments": [ "segments": [
{ {
"segment_id": 1, "segment_id": 1,
"collection_id": 100, "collection_id": 1,
"partition_id": 10, "partition_id": 1,
"channel": "channel1", "channel": "channel1",
"num_of_rows": 1000, "num_of_rows": 1000,
"state": "flushed", "state": "Growing",
"is_importing": false, "is_importing": false,
"compacted": false, "compacted": false,
"level": "L1", "level": "L1",
"is_sorted": true, "is_sorted": true,
"node_id": 1 "node_id": 1
},
{
"segment_id": 3,
"collection_id": 2,
"partition_id": 2,
"channel": "channel2",
"num_of_rows": 2000,
"state": "Growing",
"is_importing": true,
"compacted": true,
"level": "L2",
"is_sorted": false,
"node_id": 2
} }
], ],
"dm_channels": [ "dm_channels": [
@ -737,12 +1064,23 @@ var dc_dist = `
"dropped_segment_ids": [7, 8, 9], "dropped_segment_ids": [7, 8, 9],
"watch_state": "success", "watch_state": "success",
"start_watch_ts": 123456789 "start_watch_ts": 123456789
},
{
"node_id": 1,
"version": 1,
"collection_id": 100,
"channel_name": "channel3",
"unflushed_segment_ids": [1, 2, 3],
"flushed_segment_ids": [4, 5, 6],
"dropped_segment_ids": [7, 8, 9],
"watch_state": "to_watch",
"start_watch_ts": 123456789
} }
] ]
} }
` `;
var dc_build_index_task = ` const dc_build_index_task = `
[ [
{ {
"index_id": 1, "index_id": 1,
@ -767,7 +1105,7 @@ var dc_build_index_task = `
} }
]` ]`
var dc_compaction_task = ` const dc_compaction_task = `
[ [
{ {
"plan_id": 1, "plan_id": 1,
@ -795,7 +1133,7 @@ var dc_compaction_task = `
} }
]` ]`
var dn_sync_task = ` const dn_sync_task = `
[ [
{ {
"segment_id": 1, "segment_id": 1,
@ -822,7 +1160,7 @@ var dn_sync_task = `
] ]
` `
var dc_import_task = ` const dc_import_task = `
[ [
{ {
"job_id": 1, "job_id": 1,
@ -840,9 +1178,9 @@ var dc_import_task = `
"task_id": 6, "task_id": 6,
"collection_id": 7, "collection_id": 7,
"node_id": 8, "node_id": 8,
"state": "ImportTaskStateCompleted", "state": "Completed",
"reason": "", "reason": "",
"task_type": "Completed", "task_type": "ImportTask",
"created_time": "2023-10-01T00:00:00Z", "created_time": "2023-10-01T00:00:00Z",
"complete_time": "2023-10-01T01:00:00Z" "complete_time": "2023-10-01T01:00:00Z"
}, },
@ -860,7 +1198,7 @@ var dc_import_task = `
] ]
` `
var dn_segments = ` const dn_segments = `
[ [
{ {
"segment_id": 1, "segment_id": 1,
@ -868,7 +1206,7 @@ var dn_segments = `
"partition_id": 1, "partition_id": 1,
"channel": "channel1", "channel": "channel1",
"num_of_rows": 1000, "num_of_rows": 1000,
"state": "active", "state": "Growing",
"is_importing": false, "is_importing": false,
"compacted": false, "compacted": false,
"level": "L1", "level": "L1",
@ -884,7 +1222,7 @@ var dn_segments = `
"partition_id": 2, "partition_id": 2,
"channel": "channel2", "channel": "channel2",
"num_of_rows": 2000, "num_of_rows": 2000,
"state": "inactive", "state": "Sealed",
"is_importing": true, "is_importing": true,
"compacted": true, "compacted": true,
"level": "L2", "level": "L2",
@ -897,7 +1235,7 @@ var dn_segments = `
] ]
` `
var dn_channels = ` const dn_channels = `
[ [
{ {
"name": "channel1", "name": "channel1",
@ -919,3 +1257,114 @@ var dn_channels = `
} }
] ]
` `
const slowQueries = `[
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.1s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl1"],
"search_params": ["param2=value2"],
"nq": [10]
}
],
"output_fields": "field1,field2"
},
"type": "Search",
"trace_id": "729b10a6a7f32ddd7ab5c16dd30f60dc",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.2s",
"user": "test_user",
"query_params": {
"expr": "expr1",
"output_fields": "field1,field2"
},
"type": "Query",
"trace_id": "232955b7f33b135708d34c3c761b57e7",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.3s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl2"],
"search_params": ["param3=value3"],
"nq": [20]
}
],
"output_fields": "field3,field4"
},
"type": "HybridSearch",
"trace_id": "3a4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.4s",
"user": "test_user",
"query_params": {
"expr": "expr2",
"output_fields": "field5,field6"
},
"type": "Query",
"trace_id": "4b5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k",
"time": "2024-11-05 08:14:05"
},
{
"role": "proxy",
"database": "test_db",
"collection": "test_collection",
"partitions": "partition1,partition2",
"consistency_level": "Bounded",
"use_default_consistency": true,
"guarantee_timestamp": 123456789,
"duration": "1.5s",
"user": "test_user",
"query_params": {
"search_params": [
{
"dsl": ["dsl3"],
"search_params": ["param4=value4"],
"nq": [30]
}
],
"output_fields": "field7,field8"
},
"type": "Search",
"trace_id": "5c6d7e8f9a0b1c2d3e4f5g6h7i8j9k0l",
"time": "2024-11-05 08:14:05"
}
]`;

1334
internal/http/webui/static/js/render.js
View File

File diff suppressed because it is too large Load Diff

162
internal/http/webui/tasks.html
View File

@ -25,123 +25,29 @@
<h2> <h2>
QueryCoord Tasks QueryCoord Tasks
</h2> </h2>
<table id="channelCP" class="table table-hover"> <table id="qcTasks" class="table table-hover"></table>
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Source</th>
<th scope="col">Actions</th>
<th scope="col">Type</th>
<th scope="col">State</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>indexChecker</td>
<td>querynode1</td>
<td>querynode2</td>
<td>queued</td>
</tr>
<tr>
<td>2</td>
<td>loadChannel</td>
<td>querynode1</td>
<td>querynode2</td>
<td>queued</td>
</tr>
</tbody>
</table>
<h2> <h2>
Compaction Tasks Compaction Tasks
</h2> </h2>
<table id="compactionTasks" class="table table-hover"> <table id="compactionTasks" class="table table-hover"></table>
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Plan</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<h2>
Flush Tasks
</h2>
<table id="flushTasks" class="table table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Task ID</th>
<th scope="col">Segment ID</th>
<th scope="col">Insertion Size(MB)</th>
<th scope="col">Deletion Size(MB)</th>
<th scope="col">State</th>
<th scope="col">Datanode</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>1111</td>
<td>10</td>
<td>5</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>2222</td>
<td>5</td>
<td>5</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
</table>
<h2> <h2>
Index Build Tasks Index Build Tasks
</h2> </h2>
<table id="indexBuildTasks" class="table table-hover"> <table id="buildIndexTasks" class="table table-hover"></table>
<thead class="thead-light">
<tr> <h2>
<th scope="col">Task ID</th> Import Tasks
<th scope="col">Plan</th> </h2>
<th scope="col">State</th> <table id="importTasks" class="table table-hover"></table>
<th scope="col">Indexnode</th>
</tr> <h2>
</thead> Sync Task
<tbody> </h2>
<tr> <table id="syncTasks" class="table table-hover">
<td>1</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode1</td>
</tr>
<tr>
<td>2</td>
<td>fake-plan</td>
<td>running</td>
<td>datanode2</td>
</tr>
</tbody>
</table> </table>
</div> </div>
<div class="col-md-2"> <div class="col-md-2">
</div> </div>
@ -156,13 +62,47 @@
}); });
document.addEventListener("DOMContentLoaded", function() { document.addEventListener("DOMContentLoaded", function() {
fetchData(MILVUS_URI + "/", sysmetrics) fetchData(MILVUS_URI + "/_qc/tasks", qcTasks)
.then(data => { .then(data => {
//TODO add tasks render renderQCTasks(data)
}) })
.catch(error => { .catch(error => {
handleError(new Error("Unimplemented API")); handleError(error);
}); });
fetchData(MILVUS_URI + "/_dc/tasks/compaction", dc_compaction_task)
.then(data => {
renderCompactionTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dc/tasks/build_index", dc_build_index_task)
.then(data => {
renderBuildIndexTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dc/tasks/import", dc_import_task)
.then(data => {
renderImportTasks(data)
})
.catch(error => {
handleError(error);
});
fetchData(MILVUS_URI + "/_dn/tasks/sync", dn_sync_task)
.then(data => {
renderSyncTasks(data)
})
.catch(error => {
handleError(error);
});
}); });
</script> </script>
</body> </body>

52
internal/http/webui/tools.html Normal file
View File

@ -0,0 +1,52 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>tools</title>
<meta name="description" content="Milvus Management WebUI">
<link href="./static/css/bootstrap.min.css" rel="stylesheet">
<link href="./static/css/style.css" rel="stylesheet">
<script src="./static/js/jquery.min.js"></script>
<script src="./static/js/bootstrap.min.js"></script>
<script src="./static/js/bootstrap.bundle.min.js"></script>
<script src="./static/js/render.js"></script>
<script src="./static/js/common.js"></script>
<script src="./static/js/mockdata.js"></script>
</head>
<body>
<div class="container-fluid">
<div id="header"></div>
<div class="row">
<div class="col-md-2">
</div>
<div class="col-md-8">
<div class="row" style="height: 100px;"></div>
<!-- Centered Links Section -->
<div class="row text-center mb-3">
<div class="col">
<a href="#link1" class="btn btn-link" style="font-size: 1.5em;">Pprof</a>
</div>
</div>
<div class="row text-center mb-3">
<div class="col">
<a href="#link2" class="btn btn-link" style="font-size: 1.5em;">Memory Data Visualization</a>
</div>
</div>
</div>
<div class="col-md-2">
</div>
</div>
<div id="footer"></div>
</div>
<script>
$(document).ready(function(){
$('#header').load("header.html");
$('#footer').load("footer.html");
});
</script>
</body>
</html>

14
internal/proxy/http_req_impl.go
View File

@ -118,6 +118,20 @@ func getDependencies(c *gin.Context) {
c.Data(http.StatusOK, contentType, ret) c.Data(http.StatusOK, contentType, ret)
} }
func getSlowQuery(node *Proxy) gin.HandlerFunc {
return func(c *gin.Context) {
slowQueries := node.slowQueries.Values()
ret, err := json.Marshal(slowQueries)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
mhttp.HTTPReturnMessage: err.Error(),
})
return
}
c.Data(http.StatusOK, contentType, ret)
}
}
// buildReqParams fetch all parameters from query parameter of URL, add them into a map data structure. // buildReqParams fetch all parameters from query parameter of URL, add them into a map data structure.
// put key and value from query parameter into map, concatenate values with separator if values size is greater than 1 // put key and value from query parameter into map, concatenate values with separator if values size is greater than 1
func buildReqParams(c *gin.Context, metricsType string) map[string]interface{} { func buildReqParams(c *gin.Context, metricsType string) map[string]interface{} {

33
internal/proxy/impl.go
View File

@ -30,6 +30,7 @@ import (
"github.com/samber/lo" "github.com/samber/lo"
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
"go.opentelemetry.io/otel" "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
@ -3020,6 +3021,14 @@ func (node *Proxy) search(ctx context.Context, request *milvuspb.SearchRequest,
strconv.FormatInt(paramtable.GetNodeID(), 10), strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.SearchLabel, metrics.SearchLabel,
).Inc() ).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(request, user, span, traceID))
}
} }
}() }()
@ -3230,6 +3239,14 @@ func (node *Proxy) hybridSearch(ctx context.Context, request *milvuspb.HybridSea
strconv.FormatInt(paramtable.GetNodeID(), 10), strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.HybridSearchLabel, metrics.HybridSearchLabel,
).Inc() ).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithSearchRequest(newSearchReq, user, span, traceID))
}
} }
}() }()
@ -3471,7 +3488,7 @@ func (node *Proxy) Flush(ctx context.Context, request *milvuspb.FlushRequest) (*
} }
// Query get the records by primary keys. // Query get the records by primary keys.
func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryResults, error) { func (node *Proxy) query(ctx context.Context, qt *queryTask, sp trace.Span) (*milvuspb.QueryResults, error) {
request := qt.request request := qt.request
method := "Query" method := "Query"
@ -3513,6 +3530,15 @@ func (node *Proxy) query(ctx context.Context, qt *queryTask) (*milvuspb.QueryRes
strconv.FormatInt(paramtable.GetNodeID(), 10), strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.QueryLabel, metrics.QueryLabel,
).Inc() ).Inc()
user, _ := GetCurUserFromContext(ctx)
traceID := ""
if sp != nil {
traceID = sp.SpanContext().TraceID().String()
}
if node.slowQueries != nil {
node.slowQueries.Add(qt.BeginTs(), metricsinfo.NewSlowQueryWithQueryRequest(request, user, span, traceID))
}
} }
}() }()
@ -3628,7 +3654,7 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (*
request.GetCollectionName(), request.GetCollectionName(),
).Inc() ).Inc()
res, err := node.query(ctx, qt) res, err := node.query(ctx, qt, sp)
if err != nil || !merr.Ok(res.Status) { if err != nil || !merr.Ok(res.Status) {
return res, err return res, err
} }
@ -6484,6 +6510,9 @@ func (node *Proxy) RegisterRestRouter(router gin.IRouter) {
// Hook request that executed by proxy // Hook request that executed by proxy
router.GET(http.HookConfigsPath, getConfigs(paramtable.GetHookParams().GetAll())) router.GET(http.HookConfigsPath, getConfigs(paramtable.GetHookParams().GetAll()))
// Slow query request that executed by proxy
router.GET(http.SlowQueryPath, getSlowQuery(node))
// QueryCoord requests that are forwarded from proxy // QueryCoord requests that are forwarded from proxy
router.GET(http.QCTargetPath, getQueryComponentMetrics(node, metricsinfo.QueryTarget)) router.GET(http.QCTargetPath, getQueryComponentMetrics(node, metricsinfo.QueryTarget))
router.GET(http.QCDistPath, getQueryComponentMetrics(node, metricsinfo.QueryDist)) router.GET(http.QCDistPath, getQueryComponentMetrics(node, metricsinfo.QueryDist))

4
internal/proxy/proxy.go
View File

@ -26,6 +26,7 @@ import (
"time" "time"
"github.com/cockroachdb/errors" "github.com/cockroachdb/errors"
"github.com/hashicorp/golang-lru/v2/expirable"
clientv3 "go.etcd.io/etcd/client/v3" clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/atomic" "go.uber.org/atomic"
"go.uber.org/zap" "go.uber.org/zap"
@ -130,6 +131,8 @@ type Proxy struct {
// delete rate limiter // delete rate limiter
enableComplexDeleteLimit bool enableComplexDeleteLimit bool
slowQueries *expirable.LRU[Timestamp, *metricsinfo.SlowQuery]
} }
// NewProxy returns a Proxy struct. // NewProxy returns a Proxy struct.
@ -152,6 +155,7 @@ func NewProxy(ctx context.Context, factory dependency.Factory) (*Proxy, error) {
lbPolicy: lbPolicy, lbPolicy: lbPolicy,
resourceManager: resourceManager, resourceManager: resourceManager,
replicateStreamManager: replicateStreamManager, replicateStreamManager: replicateStreamManager,
slowQueries: expirable.NewLRU[Timestamp, *metricsinfo.SlowQuery](20, nil, time.Minute*15),
} }
node.UpdateStateCode(commonpb.StateCode_Abnormal) node.UpdateStateCode(commonpb.StateCode_Abnormal)
expr.Register("proxy", node) expr.Register("proxy", node)

7
internal/proxy/task_search.go
View File

@ -9,6 +9,7 @@ import (
"github.com/cockroachdb/errors" "github.com/cockroachdb/errors"
"github.com/samber/lo" "github.com/samber/lo"
"go.opentelemetry.io/otel" "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap" "go.uber.org/zap"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
@ -734,7 +735,7 @@ func (t *searchTask) PostExecute(ctx context.Context) error {
t.fillInFieldInfo() t.fillInFieldInfo()
if t.requery { if t.requery {
err = t.Requery() err = t.Requery(sp)
if err != nil { if err != nil {
log.Warn("failed to requery", zap.Error(err)) log.Warn("failed to requery", zap.Error(err))
return err return err
@ -819,7 +820,7 @@ func (t *searchTask) estimateResultSize(nq int64, topK int64) (int64, error) {
//return int64(sizePerRecord) * nq * topK, nil //return int64(sizePerRecord) * nq * topK, nil
} }
func (t *searchTask) Requery() error { func (t *searchTask) Requery(span trace.Span) error {
queryReq := &milvuspb.QueryRequest{ queryReq := &milvuspb.QueryRequest{
Base: &commonpb.MsgBase{ Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Retrieve, MsgType: commonpb.MsgType_Retrieve,
@ -864,7 +865,7 @@ func (t *searchTask) Requery() error {
fastSkip: true, fastSkip: true,
reQuery: true, reQuery: true,
} }
queryResult, err := t.node.(*Proxy).query(t.ctx, qt) queryResult, err := t.node.(*Proxy).query(t.ctx, qt, span)
if err != nil { if err != nil {
return err return err
} }

6
internal/proxy/task_search_test.go
View File

@ -2744,7 +2744,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node, node: node,
} }
err := qt.Requery() err := qt.Requery(nil)
assert.NoError(t, err) assert.NoError(t, err)
assert.Len(t, qt.result.Results.FieldsData, 2) assert.Len(t, qt.result.Results.FieldsData, 2)
for _, field := range qt.result.Results.FieldsData { for _, field := range qt.result.Results.FieldsData {
@ -2773,7 +2773,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node, node: node,
} }
err := qt.Requery() err := qt.Requery(nil)
t.Logf("err = %s", err) t.Logf("err = %s", err)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -2807,7 +2807,7 @@ func TestSearchTask_Requery(t *testing.T) {
node: node, node: node,
} }
err := qt.Requery() err := qt.Requery(nil)
t.Logf("err = %s", err) t.Logf("err = %s", err)
assert.Error(t, err) assert.Error(t, err)
}) })

4
internal/querycoordv2/meta/dist_manager.go
View File

@ -48,10 +48,6 @@ func (dm *DistributionManager) GetDistributionJSON() string {
channels := dm.GetChannelDist() channels := dm.GetChannelDist()
leaderView := dm.GetLeaderView() leaderView := dm.GetLeaderView()
if len(segments) == 0 && len(channels) == 0 && len(leaderView) == 0 {
return ""
}
dist := &metricsinfo.QueryCoordDist{ dist := &metricsinfo.QueryCoordDist{
Segments: segments, Segments: segments,
DMChannels: channels, DMChannels: channels,

9
internal/querycoordv2/server.go
View File

@ -200,11 +200,16 @@ func (s *Server) registerMetricsRequest() {
} }
QueryDistAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { QueryDistAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {
return s.targetMgr.GetTargetJSON(meta.CurrentTarget), nil return s.dist.GetDistributionJSON(), nil
} }
QueryTargetAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { QueryTargetAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {
return s.dist.GetDistributionJSON(), nil scope := meta.CurrentTarget
v := jsonReq.Get(metricsinfo.MetricRequestParamTargetScopeKey)
if v.Exists() {
scope = meta.TargetScope(v.Int())
}
return s.targetMgr.GetTargetJSON(scope), nil
} }
QueryReplicasAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) { QueryReplicasAction := func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) {

6
internal/querycoordv2/task/scheduler.go
View File

@ -202,7 +202,7 @@ func NewScheduler(ctx context.Context,
channelTasks: make(map[replicaChannelIndex]Task), channelTasks: make(map[replicaChannelIndex]Task),
processQueue: newTaskQueue(), processQueue: newTaskQueue(),
waitQueue: newTaskQueue(), waitQueue: newTaskQueue(),
taskStats: expirable.NewLRU[UniqueID, Task](512, nil, time.Minute*30), taskStats: expirable.NewLRU[UniqueID, Task](64, nil, time.Minute*15),
} }
} }
@ -574,10 +574,6 @@ func (scheduler *taskScheduler) GetSegmentTaskNum() int {
// GetTasksJSON returns the JSON string of all tasks. // GetTasksJSON returns the JSON string of all tasks.
// the task stats object is thread safe and can be accessed without lock // the task stats object is thread safe and can be accessed without lock
func (scheduler *taskScheduler) GetTasksJSON() string { func (scheduler *taskScheduler) GetTasksJSON() string {
if scheduler.taskStats.Len() == 0 {
return ""
}
tasks := scheduler.taskStats.Values() tasks := scheduler.taskStats.Values()
ret, err := json.Marshal(tasks) ret, err := json.Marshal(tasks)
if err != nil { if err != nil {

5
pkg/go.mod
View File

@ -26,6 +26,7 @@ require (
github.com/shirou/gopsutil/v3 v3.22.9 github.com/shirou/gopsutil/v3 v3.22.9
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.0
github.com/spaolacci/murmur3 v1.1.0 github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.0
github.com/streamnative/pulsarctl v0.5.0 github.com/streamnative/pulsarctl v0.5.0
github.com/stretchr/testify v1.9.0 github.com/stretchr/testify v1.9.0
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
@ -55,7 +56,7 @@ require (
google.golang.org/grpc v1.65.0 google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2 google.golang.org/protobuf v1.34.2
gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v2 v2.4.0
k8s.io/apimachinery v0.28.6 k8s.io/apimachinery v0.28.6
) )
@ -169,7 +170,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect
) )

1
pkg/go.sum
View File

@ -655,6 +655,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=

2
pkg/util/metricsinfo/metric_request.go
View File

@ -88,6 +88,8 @@ const (
// MetricRequestParamVerboseKey as a request parameter decide to whether return verbose value // MetricRequestParamVerboseKey as a request parameter decide to whether return verbose value
MetricRequestParamVerboseKey = "verbose" MetricRequestParamVerboseKey = "verbose"
MetricRequestParamTargetScopeKey = "target_scope"
) )
type MetricsRequestAction func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error) type MetricsRequestAction func(ctx context.Context, req *milvuspb.GetMetricsRequest, jsonReq gjson.Result) (string, error)

48
pkg/util/metricsinfo/metrics_info.go
View File

@ -70,6 +70,34 @@ const (
MilvusUsedGoVersion = "MILVUS_USED_GO_VERSION" MilvusUsedGoVersion = "MILVUS_USED_GO_VERSION"
) )
type SearchParams struct {
DSL []string `json:"dsl,omitempty"`
SearchParams []string `json:"search_params,omitempty"`
NQ []int64 `json:"nq,omitempty"`
}
type QueryParams struct {
SearchParams []*SearchParams `json:"search_params,omitempty"`
Expr string `json:"expr,omitempty"`
OutputFields string `json:"output_fields,omitempty"`
}
type SlowQuery struct {
Time string `json:"time,omitempty"`
Role string `json:"role,omitempty"`
Database string `json:"database,omitempty"`
Collection string `json:"collection,omitempty"`
Partitions string `json:"partitions,omitempty"`
ConsistencyLevel string `json:"consistency_level,omitempty"`
UseDefaultConsistency bool `json:"use_default_consistency,omitempty"`
GuaranteeTimestamp uint64 `json:"guarantee_timestamp,omitempty"`
Duration string `json:"duration,omitempty"`
User string `json:"user,omitempty"`
QueryParams *QueryParams `json:"query_params,omitempty"`
Type string `json:"type,omitempty"`
TraceID string `json:"trace_id,omitempty"`
}
type DmChannel struct { type DmChannel struct {
NodeID int64 `json:"node_id,omitempty"` NodeID int64 `json:"node_id,omitempty"`
Version int64 `json:"version,omitempty"` Version int64 `json:"version,omitempty"`
@ -109,7 +137,6 @@ type Segment struct {
FlushedRows int64 `json:"flushed_rows,omitempty"` FlushedRows int64 `json:"flushed_rows,omitempty"`
SyncBufferRows int64 `json:"sync_buffer_rows,omitempty"` SyncBufferRows int64 `json:"sync_buffer_rows,omitempty"`
SyncingRows int64 `json:"syncing_rows,omitempty"` SyncingRows int64 `json:"syncing_rows,omitempty"`
// TODO add checkpoints
} }
type SegmentIndex struct { type SegmentIndex struct {
@ -127,15 +154,16 @@ type QueryCoordTarget struct {
} }
type LeaderView struct { type LeaderView struct {
LeaderID int64 `json:"leader_id"` LeaderID int64 `json:"leader_id,omitempty"`
CollectionID int64 `json:"collection_id"` CollectionID int64 `json:"collection_id,omitempty"`
Channel string `json:"channel"` NodeID int64 `json:"node_id,omitempty"`
Version int64 `json:"version"` Channel string `json:"channel,omitempty"`
SealedSegments []*Segment `json:"sealed_segments"` Version int64 `json:"version,omitempty"`
GrowingSegments []*Segment `json:"growing_segments"` SealedSegments []*Segment `json:"sealed_segments,omitempty"`
TargetVersion int64 `json:"target_version"` GrowingSegments []*Segment `json:"growing_segments,omitempty"`
NumOfGrowingRows int64 `json:"num_of_growing_rows"` TargetVersion int64 `json:"target_version,omitempty"`
UnServiceableError string `json:"unserviceable_error"` NumOfGrowingRows int64 `json:"num_of_growing_rows,omitempty"`
UnServiceableError string `json:"unserviceable_error,omitempty"`
} }
type QueryCoordDist struct { type QueryCoordDist struct {

88
pkg/util/metricsinfo/utils.go
View File

@ -14,10 +14,15 @@ package metricsinfo
import ( import (
"encoding/json" "encoding/json"
"os" "os"
"strings"
"time"
"go.uber.org/zap" "go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
) )
// FillDeployMetricsWithEnv fill deploy metrics with env. // FillDeployMetricsWithEnv fill deploy metrics with env.
@ -34,10 +39,6 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) {
return "", err return "", err
} }
if len(metrics) == 0 {
return "", nil
}
bs, err := json.Marshal(metrics) bs, err := json.Marshal(metrics)
if err != nil { if err != nil {
log.Warn("marshal metrics value failed", zap.Any("metrics", metrics), zap.String("err", err.Error())) log.Warn("marshal metrics value failed", zap.Any("metrics", metrics), zap.String("err", err.Error()))
@ -45,3 +46,82 @@ func MarshalGetMetricsValues[T any](metrics []T, err error) (string, error) {
} }
return string(bs), nil return string(bs), nil
} }
func getSearchParamString(params []*commonpb.KeyValuePair) string {
searchParams := ""
for _, kv := range params {
searchParams += kv.Key + "=" + kv.Value + ","
}
if len(searchParams) > 0 {
searchParams = searchParams[:len(searchParams)-1]
}
return searchParams
}
func NewSlowQueryWithQueryRequest(request *milvuspb.QueryRequest, user string, cost time.Duration, traceID string) *SlowQuery {
queryParams := &QueryParams{
Expr: request.GetExpr(),
OutputFields: strings.Join(request.GetOutputFields(), ","),
}
return &SlowQuery{
Role: typeutil.ProxyRole,
Database: request.GetDbName(),
Collection: request.GetCollectionName(),
Partitions: strings.Join(request.GetPartitionNames(), ","),
ConsistencyLevel: request.GetConsistencyLevel().String(),
UseDefaultConsistency: request.GetUseDefaultConsistency(),
GuaranteeTimestamp: request.GetGuaranteeTimestamp(),
Duration: cost.String(),
User: user,
QueryParams: queryParams,
Type: "Query",
TraceID: traceID,
Time: time.Now().Format(time.DateTime),
}
}
func NewSlowQueryWithSearchRequest(request *milvuspb.SearchRequest, user string, cost time.Duration, traceID string) *SlowQuery {
searchParams := getSearchParamString(request.GetSearchParams())
var subReqs []*SearchParams
for _, req := range request.GetSubReqs() {
subReqs = append(subReqs, &SearchParams{
DSL: []string{req.GetDsl()},
SearchParams: []string{getSearchParamString(req.GetSearchParams())},
NQ: []int64{req.GetNq()},
})
}
searchType := "HybridSearch"
if len(request.GetSubReqs()) == 0 {
subReqs = append(subReqs, &SearchParams{
DSL: []string{request.GetDsl()},
SearchParams: []string{searchParams},
NQ: []int64{request.GetNq()},
})
searchType = "Search"
}
queryParams := &QueryParams{
SearchParams: subReqs,
Expr: request.GetDsl(),
OutputFields: strings.Join(request.GetOutputFields(), ","),
}
return &SlowQuery{
Role: typeutil.ProxyRole,
Database: request.GetDbName(),
Collection: request.GetCollectionName(),
Partitions: strings.Join(request.GetPartitionNames(), ","),
ConsistencyLevel: request.GetConsistencyLevel().String(),
UseDefaultConsistency: request.GetUseDefaultConsistency(),
GuaranteeTimestamp: request.GetGuaranteeTimestamp(),
Duration: cost.String(),
User: user,
QueryParams: queryParams,
Type: searchType,
TraceID: traceID,
Time: time.Now().Format(time.DateTime),
}
}

75
pkg/util/metricsinfo/utils_test.go
View File

@ -13,8 +13,12 @@ package metricsinfo
import ( import (
"testing" "testing"
"time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
) )
func TestFillDeployMetricsWithEnv(t *testing.T) { func TestFillDeployMetricsWithEnv(t *testing.T) {
@ -43,3 +47,74 @@ func TestFillDeployMetricsWithEnv(t *testing.T) {
assert.Equal(t, goVersion, m.UsedGoVersion) assert.Equal(t, goVersion, m.UsedGoVersion)
assert.Equal(t, buildTime, m.BuildTime) assert.Equal(t, buildTime, m.BuildTime)
} }
func TestNewSlowQueryWithSearchRequest(t *testing.T) {
request := &milvuspb.SearchRequest{
DbName: "test_db",
CollectionName: "test_collection",
PartitionNames: []string{"partition1", "partition2"},
ConsistencyLevel: commonpb.ConsistencyLevel_Bounded,
UseDefaultConsistency: true,
GuaranteeTimestamp: 123456789,
SearchParams: []*commonpb.KeyValuePair{{Key: "param1", Value: "value1"}},
SubReqs: []*milvuspb.SubSearchRequest{{Dsl: "dsl1", SearchParams: []*commonpb.KeyValuePair{{Key: "param2", Value: "value2"}}, Nq: 10}},
Dsl: "dsl2",
Nq: 20,
OutputFields: []string{"field1", "field2"},
}
user := "test_user"
cost := time.Duration(100) * time.Millisecond
slowQuery := NewSlowQueryWithSearchRequest(request, user, cost, "")
assert.NotNil(t, slowQuery)
assert.Equal(t, "proxy", slowQuery.Role)
assert.Equal(t, "test_db", slowQuery.Database)
assert.Equal(t, "test_collection", slowQuery.Collection)
assert.Equal(t, "partition1,partition2", slowQuery.Partitions)
assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel)
assert.True(t, slowQuery.UseDefaultConsistency)
assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp)
assert.Equal(t, "100ms", slowQuery.Duration)
assert.Equal(t, user, slowQuery.User)
assert.Equal(t, "HybridSearch", slowQuery.Type)
assert.NotNil(t, slowQuery.QueryParams)
assert.Equal(t, "dsl2", slowQuery.QueryParams.Expr)
assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields)
assert.Len(t, slowQuery.QueryParams.SearchParams, 1)
assert.Equal(t, []string{"dsl1"}, slowQuery.QueryParams.SearchParams[0].DSL)
assert.Equal(t, []string{"param2=value2"}, slowQuery.QueryParams.SearchParams[0].SearchParams)
assert.Equal(t, []int64{10}, slowQuery.QueryParams.SearchParams[0].NQ)
}
func TestNewSlowQueryWithQueryRequest(t *testing.T) {
request := &milvuspb.QueryRequest{
DbName: "test_db",
CollectionName: "test_collection",
PartitionNames: []string{"partition1", "partition2"},
ConsistencyLevel: commonpb.ConsistencyLevel_Bounded,
UseDefaultConsistency: true,
GuaranteeTimestamp: 123456789,
Expr: "expr1",
OutputFields: []string{"field1", "field2"},
}
user := "test_user"
cost := time.Duration(100) * time.Millisecond
slowQuery := NewSlowQueryWithQueryRequest(request, user, cost, "")
assert.NotNil(t, slowQuery)
assert.Equal(t, "proxy", slowQuery.Role)
assert.Equal(t, "test_db", slowQuery.Database)
assert.Equal(t, "test_collection", slowQuery.Collection)
assert.Equal(t, "partition1,partition2", slowQuery.Partitions)
assert.Equal(t, "Bounded", slowQuery.ConsistencyLevel)
assert.True(t, slowQuery.UseDefaultConsistency)
assert.Equal(t, uint64(123456789), slowQuery.GuaranteeTimestamp)
assert.Equal(t, "100ms", slowQuery.Duration)
assert.Equal(t, user, slowQuery.User)
assert.Equal(t, "Query", slowQuery.Type)
assert.NotNil(t, slowQuery.QueryParams)
assert.Equal(t, "expr1", slowQuery.QueryParams.Expr)
assert.Equal(t, "field1,field2", slowQuery.QueryParams.OutputFields)
}

9
tests/integration/minicluster_v2.go
View File

@ -464,14 +464,23 @@ func (cluster *MiniClusterV2) Stop() error {
if cluster.clientConn != nil { if cluster.clientConn != nil {
cluster.clientConn.Close() cluster.clientConn.Close()
} }
if cluster.RootCoord != nil {
cluster.RootCoord.Stop() cluster.RootCoord.Stop()
log.Info("mini cluster rootCoord stopped") log.Info("mini cluster rootCoord stopped")
}
if cluster.DataCoord != nil {
cluster.DataCoord.Stop() cluster.DataCoord.Stop()
log.Info("mini cluster dataCoord stopped") log.Info("mini cluster dataCoord stopped")
}
if cluster.QueryCoord != nil {
cluster.QueryCoord.Stop() cluster.QueryCoord.Stop()
log.Info("mini cluster queryCoord stopped") log.Info("mini cluster queryCoord stopped")
}
if cluster.Proxy != nil {
cluster.Proxy.Stop() cluster.Proxy.Stop()
log.Info("mini cluster proxy stopped") log.Info("mini cluster proxy stopped")
}
cluster.StopAllDataNodes() cluster.StopAllDataNodes()
cluster.StopAllStreamingNodes() cluster.StopAllStreamingNodes()