admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.5] Output index version information in the DescribeIndex interface (#41841)

Browse Source 
issue: #41431 
pr: #41847

Signed-off-by: foxspy <xianliang.li@zilliz.com>
This commit is contained in:
foxspy 2025-05-15 14:34:23 +08:00 committed by GitHub
parent 6278fb3b56
commit e36df6991b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 629 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -21,7 +21,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.18.0
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33
github.com/minio/minio-go/v7 v7.0.73
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0

4
go.sum
View File

@ -634,8 +634,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 h1:NtZRds0z4fgvbo8aZvv4Me4V9VEXNinSCwOyJJM5bOY=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33 h1:2tRZDfyqglCtU0jgCMtAtJirhnt3qgKDf4U6u4g6xsU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

9
internal/datacoord/index_meta.go
View File

@ -1072,10 +1072,11 @@ func (m *indexMeta) getSegmentsIndexStates(collectionID UniqueID, segmentIDs []U
for _, segIdx := range segIndexInfos.Values() {
if index, ok := fieldIndexes[segIdx.IndexID]; ok && !index.IsDeleted {
ret[segID][segIdx.IndexID] = &indexpb.SegmentIndexState{
SegmentID: segID,
State: segIdx.IndexState,
FailReason: segIdx.FailReason,
IndexName: index.IndexName,
SegmentID: segID,
State: segIdx.IndexState,
FailReason: segIdx.FailReason,
IndexName: index.IndexName,
IndexVersion: segIdx.CurrentIndexVersion,
}
}
}

15
internal/datacoord/index_service.go
View File

@ -19,6 +19,7 @@ package datacoord
import (
"context"
"fmt"
"math"
"strings"
"time"
@ -708,6 +709,9 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
pendingIndexRows = int64(0)
)
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for segID, seg := range segments {
if seg.state != commonpb.SegmentState_Flushed && seg.state != commonpb.SegmentState_Flushing {
continue
@ -744,6 +748,12 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
case commonpb.IndexState_Finished:
cntFinished++
indexedRows += seg.numRows
if segIdx.IndexVersion < minIndexVersion {
minIndexVersion = segIdx.IndexVersion
}
if segIdx.IndexVersion > maxIndexVersion {
maxIndexVersion = segIdx.IndexVersion
}
case commonpb.IndexState_Failed:
cntFailed++
failReason += fmt.Sprintf("%d: %s;", segID, segIdx.FailReason)
@ -757,6 +767,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
}
indexInfo.TotalRows = totalRows
indexInfo.PendingIndexRows = pendingIndexRows
indexInfo.MinIndexVersion = minIndexVersion
indexInfo.MaxIndexVersion = maxIndexVersion
switch {
case cntFailed > 0:
indexInfo.State = commonpb.IndexState_Failed
@ -772,7 +784,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
log.RatedInfo(60, "completeIndexInfo success", zap.Int64("collectionID", index.CollectionID), zap.Int64("indexID", index.IndexID),
zap.Int64("totalRows", indexInfo.TotalRows), zap.Int64("indexRows", indexInfo.IndexedRows),
zap.Int64("pendingIndexRows", indexInfo.PendingIndexRows),
zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason))
zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason),
zap.Int32("minIndexVersion", indexInfo.MinIndexVersion), zap.Int32("maxIndexVersion", indexInfo.MaxIndexVersion))
}
// GetIndexBuildProgress get the index building progress by num rows.

79
internal/datacoord/index_service_test.go
View File

@ -18,6 +18,7 @@ package datacoord
import (
"context"
"math"
"testing"
"time"
@ -1472,6 +1473,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil,
IndexSerializedSize: 0,
WriteHandoff: false,
CurrentIndexVersion: 7,
})
segIdx1.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID,
@ -1489,6 +1491,8 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil,
IndexSerializedSize: 0,
WriteHandoff: false,
// deleted index
CurrentIndexVersion: 6,
})
segIdx1.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID,
@ -1545,28 +1549,30 @@ func TestServer_DescribeIndex(t *testing.T) {
segIdx2 := typeutil.NewConcurrentMap[UniqueID, *model.SegmentIndex]()
segIdx2.Insert(indexID, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID,
BuildID: buildID,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID,
BuildID: buildID,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
segIdx2.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 1,
BuildID: buildID + 1,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 1,
BuildID: buildID + 1,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
segIdx2.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID - 1,
@ -1594,16 +1600,17 @@ func TestServer_DescribeIndex(t *testing.T) {
CreatedUTCTime: createTS,
})
segIdx2.Insert(indexID+5, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 5,
BuildID: buildID + 5,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 5,
BuildID: buildID + 5,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
s.meta.indexMeta.segmentIndexes.Insert(segID-1, segIdx2)
@ -1625,6 +1632,18 @@ func TestServer_DescribeIndex(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, 5, len(resp.GetIndexInfos()))
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for _, indexInfo := range resp.GetIndexInfos() {
if indexInfo.GetMinIndexVersion() < minIndexVersion {
minIndexVersion = indexInfo.GetMinIndexVersion()
}
if indexInfo.GetMaxIndexVersion() > maxIndexVersion {
maxIndexVersion = indexInfo.GetMaxIndexVersion()
}
}
assert.Equal(t, int32(7), minIndexVersion)
assert.Equal(t, int32(7), maxIndexVersion)
})
t.Run("describe after drop index", func(t *testing.T) {

3
internal/distributed/proxy/httpserver/constant.go
View File

@ -159,6 +159,9 @@ const (
HTTPReturnIndexState = "indexState"
HTTPReturnIndexFailReason = "failReason"
HTTPReturnMinIndexVersion = "minIndexVersion"
HTTPReturnMaxIndexVersion = "maxIndexVersion"
HTTPReturnDistance = "distance"
HTTPReturnRowCount = "rowCount"

2
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -2281,6 +2281,8 @@ func (h *HandlersV2) describeIndex(ctx context.Context, c *gin.Context, anyReq a
HTTPReturnIndexIndexedRows: indexDescription.IndexedRows,
HTTPReturnIndexState: indexDescription.State.String(),
HTTPReturnIndexFailReason: indexDescription.IndexStateFailReason,
HTTPReturnMinIndexVersion: indexDescription.MinIndexVersion,
HTTPReturnMaxIndexVersion: indexDescription.MaxIndexVersion,
}
indexInfos = append(indexInfos, indexInfo)
}

4
internal/proxy/task_index.go
View File

@ -839,6 +839,8 @@ func (dit *describeIndexTask) Execute(ctx context.Context) error {
PendingIndexRows: indexInfo.GetPendingIndexRows(),
State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
}
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
}
@ -948,6 +950,8 @@ func (dit *getIndexStatisticsTask) Execute(ctx context.Context) error {
TotalRows: indexInfo.GetTotalRows(),
State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
}
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
}

2
pkg/go.mod
View File

@ -14,7 +14,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.7
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33
github.com/nats-io/nats-server/v2 v2.10.12
github.com/nats-io/nats.go v1.34.1
github.com/panjf2000/ants/v2 v2.7.2

4
pkg/go.sum
View File

@ -488,8 +488,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 h1:NtZRds0z4fgvbo8aZvv4Me4V9VEXNinSCwOyJJM5bOY=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33 h1:2tRZDfyqglCtU0jgCMtAtJirhnt3qgKDf4U6u4g6xsU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=

3
pkg/proto/index_coord.proto
View File

@ -54,6 +54,8 @@ message IndexInfo {
bool is_auto_index = 11;
repeated common.KeyValuePair user_index_params = 12;
int64 pending_index_rows = 13;
int32 min_index_version = 14;
int32 max_index_version = 15;
}
message FieldIndex {
@ -118,6 +120,7 @@ message SegmentIndexState {
common.IndexState state = 2;
string fail_reason = 3;
string index_name = 4;
int32 index_version = 5;
}
message GetSegmentIndexStateResponse {

1054
pkg/proto/indexpb/index_coord.pb.go
View File

File diff suppressed because it is too large Load Diff