admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.5] Output index version information in the DescribeIndex interface (#41841)

Browse Source 
issue: #41431 
pr: #41847

Signed-off-by: foxspy <xianliang.li@zilliz.com>
This commit is contained in:
foxspy 2025-05-15 14:34:23 +08:00 committed by GitHub
parent 6278fb3b56
commit e36df6991b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 629 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -21,7 +21,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.18.0 github.com/klauspost/compress v1.18.0
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33
github.com/minio/minio-go/v7 v7.0.73 github.com/minio/minio-go/v7 v7.0.73
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_golang v1.14.0

4
go.sum
View File

@ -634,8 +634,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 h1:NtZRds0z4fgvbo8aZvv4Me4V9VEXNinSCwOyJJM5bOY= github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33 h1:2tRZDfyqglCtU0jgCMtAtJirhnt3qgKDf4U6u4g6xsU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE= github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

1
internal/datacoord/index_meta.go
View File

@ -1076,6 +1076,7 @@ func (m *indexMeta) getSegmentsIndexStates(collectionID UniqueID, segmentIDs []U
State: segIdx.IndexState, State: segIdx.IndexState,
FailReason: segIdx.FailReason, FailReason: segIdx.FailReason,
IndexName: index.IndexName, IndexName: index.IndexName,
IndexVersion: segIdx.CurrentIndexVersion,
} }
} }
} }

15
internal/datacoord/index_service.go
View File

@ -19,6 +19,7 @@ package datacoord
import ( import (
"context" "context"
"fmt" "fmt"
"math"
"strings" "strings"
"time" "time"
@ -708,6 +709,9 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
pendingIndexRows = int64(0) pendingIndexRows = int64(0)
) )
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for segID, seg := range segments { for segID, seg := range segments {
if seg.state != commonpb.SegmentState_Flushed && seg.state != commonpb.SegmentState_Flushing { if seg.state != commonpb.SegmentState_Flushed && seg.state != commonpb.SegmentState_Flushing {
continue continue
@ -744,6 +748,12 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
case commonpb.IndexState_Finished: case commonpb.IndexState_Finished:
cntFinished++ cntFinished++
indexedRows += seg.numRows indexedRows += seg.numRows
if segIdx.IndexVersion < minIndexVersion {
minIndexVersion = segIdx.IndexVersion
}
if segIdx.IndexVersion > maxIndexVersion {
maxIndexVersion = segIdx.IndexVersion
}
case commonpb.IndexState_Failed: case commonpb.IndexState_Failed:
cntFailed++ cntFailed++
failReason += fmt.Sprintf("%d: %s;", segID, segIdx.FailReason) failReason += fmt.Sprintf("%d: %s;", segID, segIdx.FailReason)
@ -757,6 +767,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
} }
indexInfo.TotalRows = totalRows indexInfo.TotalRows = totalRows
indexInfo.PendingIndexRows = pendingIndexRows indexInfo.PendingIndexRows = pendingIndexRows
indexInfo.MinIndexVersion = minIndexVersion
indexInfo.MaxIndexVersion = maxIndexVersion
switch { switch {
case cntFailed > 0: case cntFailed > 0:
indexInfo.State = commonpb.IndexState_Failed indexInfo.State = commonpb.IndexState_Failed
@ -772,7 +784,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
log.RatedInfo(60, "completeIndexInfo success", zap.Int64("collectionID", index.CollectionID), zap.Int64("indexID", index.IndexID), log.RatedInfo(60, "completeIndexInfo success", zap.Int64("collectionID", index.CollectionID), zap.Int64("indexID", index.IndexID),
zap.Int64("totalRows", indexInfo.TotalRows), zap.Int64("indexRows", indexInfo.IndexedRows), zap.Int64("totalRows", indexInfo.TotalRows), zap.Int64("indexRows", indexInfo.IndexedRows),
zap.Int64("pendingIndexRows", indexInfo.PendingIndexRows), zap.Int64("pendingIndexRows", indexInfo.PendingIndexRows),
zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason)) zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason),
zap.Int32("minIndexVersion", indexInfo.MinIndexVersion), zap.Int32("maxIndexVersion", indexInfo.MaxIndexVersion))
} }
// GetIndexBuildProgress get the index building progress by num rows. // GetIndexBuildProgress get the index building progress by num rows.

19
internal/datacoord/index_service_test.go
View File

@ -18,6 +18,7 @@ package datacoord
import ( import (
"context" "context"
"math"
"testing" "testing"
"time" "time"
@ -1472,6 +1473,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil, IndexFileKeys: nil,
IndexSerializedSize: 0, IndexSerializedSize: 0,
WriteHandoff: false, WriteHandoff: false,
CurrentIndexVersion: 7,
}) })
segIdx1.Insert(indexID+1, &model.SegmentIndex{ segIdx1.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID, SegmentID: segID,
@ -1489,6 +1491,8 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil, IndexFileKeys: nil,
IndexSerializedSize: 0, IndexSerializedSize: 0,
WriteHandoff: false, WriteHandoff: false,
// deleted index
CurrentIndexVersion: 6,
}) })
segIdx1.Insert(indexID+3, &model.SegmentIndex{ segIdx1.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID, SegmentID: segID,
@ -1555,6 +1559,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexVersion: 1, IndexVersion: 1,
IndexState: commonpb.IndexState_Finished, IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS, CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
}) })
segIdx2.Insert(indexID+1, &model.SegmentIndex{ segIdx2.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID - 1, SegmentID: segID - 1,
@ -1567,6 +1572,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexVersion: 1, IndexVersion: 1,
IndexState: commonpb.IndexState_Finished, IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS, CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
}) })
segIdx2.Insert(indexID+3, &model.SegmentIndex{ segIdx2.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID - 1, SegmentID: segID - 1,
@ -1604,6 +1610,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexVersion: 1, IndexVersion: 1,
IndexState: commonpb.IndexState_Finished, IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS, CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
}) })
s.meta.indexMeta.segmentIndexes.Insert(segID-1, segIdx2) s.meta.indexMeta.segmentIndexes.Insert(segID-1, segIdx2)
@ -1625,6 +1632,18 @@ func TestServer_DescribeIndex(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode()) assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, 5, len(resp.GetIndexInfos())) assert.Equal(t, 5, len(resp.GetIndexInfos()))
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for _, indexInfo := range resp.GetIndexInfos() {
if indexInfo.GetMinIndexVersion() < minIndexVersion {
minIndexVersion = indexInfo.GetMinIndexVersion()
}
if indexInfo.GetMaxIndexVersion() > maxIndexVersion {
maxIndexVersion = indexInfo.GetMaxIndexVersion()
}
}
assert.Equal(t, int32(7), minIndexVersion)
assert.Equal(t, int32(7), maxIndexVersion)
}) })
t.Run("describe after drop index", func(t *testing.T) { t.Run("describe after drop index", func(t *testing.T) {

3
internal/distributed/proxy/httpserver/constant.go
View File

@ -159,6 +159,9 @@ const (
HTTPReturnIndexState = "indexState" HTTPReturnIndexState = "indexState"
HTTPReturnIndexFailReason = "failReason" HTTPReturnIndexFailReason = "failReason"
HTTPReturnMinIndexVersion = "minIndexVersion"
HTTPReturnMaxIndexVersion = "maxIndexVersion"
HTTPReturnDistance = "distance" HTTPReturnDistance = "distance"
HTTPReturnRowCount = "rowCount" HTTPReturnRowCount = "rowCount"

2
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -2281,6 +2281,8 @@ func (h *HandlersV2) describeIndex(ctx context.Context, c *gin.Context, anyReq a
HTTPReturnIndexIndexedRows: indexDescription.IndexedRows, HTTPReturnIndexIndexedRows: indexDescription.IndexedRows,
HTTPReturnIndexState: indexDescription.State.String(), HTTPReturnIndexState: indexDescription.State.String(),
HTTPReturnIndexFailReason: indexDescription.IndexStateFailReason, HTTPReturnIndexFailReason: indexDescription.IndexStateFailReason,
HTTPReturnMinIndexVersion: indexDescription.MinIndexVersion,
HTTPReturnMaxIndexVersion: indexDescription.MaxIndexVersion,
} }
indexInfos = append(indexInfos, indexInfo) indexInfos = append(indexInfos, indexInfo)
} }

4
internal/proxy/task_index.go
View File

@ -839,6 +839,8 @@ func (dit *describeIndexTask) Execute(ctx context.Context) error {
PendingIndexRows: indexInfo.GetPendingIndexRows(), PendingIndexRows: indexInfo.GetPendingIndexRows(),
State: indexInfo.GetState(), State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(), IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
} }
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc) dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
} }
@ -948,6 +950,8 @@ func (dit *getIndexStatisticsTask) Execute(ctx context.Context) error {
TotalRows: indexInfo.GetTotalRows(), TotalRows: indexInfo.GetTotalRows(),
State: indexInfo.GetState(), State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(), IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
} }
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc) dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
} }

2
pkg/go.mod
View File

@ -14,7 +14,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/json-iterator/go v1.1.12 github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.7 github.com/klauspost/compress v1.17.7
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33
github.com/nats-io/nats-server/v2 v2.10.12 github.com/nats-io/nats-server/v2 v2.10.12
github.com/nats-io/nats.go v1.34.1 github.com/nats-io/nats.go v1.34.1
github.com/panjf2000/ants/v2 v2.7.2 github.com/panjf2000/ants/v2 v2.7.2

4
pkg/go.sum
View File

@ -488,8 +488,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156 h1:NtZRds0z4fgvbo8aZvv4Me4V9VEXNinSCwOyJJM5bOY= github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33 h1:2tRZDfyqglCtU0jgCMtAtJirhnt3qgKDf4U6u4g6xsU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250429023443-1bb3da2c9156/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/milvus-proto/go-api/v2 v2.5.12-0.20250514091616-5b2150d3bf33/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE= github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=

3
pkg/proto/index_coord.proto
View File

@ -54,6 +54,8 @@ message IndexInfo {
bool is_auto_index = 11; bool is_auto_index = 11;
repeated common.KeyValuePair user_index_params = 12; repeated common.KeyValuePair user_index_params = 12;
int64 pending_index_rows = 13; int64 pending_index_rows = 13;
int32 min_index_version = 14;
int32 max_index_version = 15;
} }
message FieldIndex { message FieldIndex {
@ -118,6 +120,7 @@ message SegmentIndexState {
common.IndexState state = 2; common.IndexState state = 2;
string fail_reason = 3; string fail_reason = 3;
string index_name = 4; string index_name = 4;
int32 index_version = 5;
} }
message GetSegmentIndexStateResponse { message GetSegmentIndexStateResponse {

1046
pkg/proto/indexpb/index_coord.pb.go
View File

File diff suppressed because it is too large Load Diff