admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Output index version information in the DescribeIndex interface (#41847)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/41431

Signed-off-by: xianliang.li <xianliang.li@zilliz.com>
This commit is contained in:
foxspy 2025-05-15 14:36:22 +08:00 committed by GitHub
parent b59a2d669f
commit 1c794be119
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 615 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -21,7 +21,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.9
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad
github.com/minio/minio-go/v7 v7.0.73
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0

4
go.sum
View File

@ -739,8 +739,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030 h1:TZR8QUxLExFI/LBexz2/ucWGvvDZv73IXb/dUNlwRrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad h1:sjhse0GoiU4ktY+aKSNeWg/zBS5jXvPFx818wGAVpZs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

9
internal/datacoord/index_meta.go
View File

@ -1068,10 +1068,11 @@ func (m *indexMeta) getSegmentsIndexStates(collectionID UniqueID, segmentIDs []U
for _, segIdx := range segIndexInfos.Values() {
if index, ok := fieldIndexes[segIdx.IndexID]; ok && !index.IsDeleted {
ret[segID][segIdx.IndexID] = &indexpb.SegmentIndexState{
SegmentID: segID,
State: segIdx.IndexState,
FailReason: segIdx.FailReason,
IndexName: index.IndexName,
SegmentID: segID,
State: segIdx.IndexState,
FailReason: segIdx.FailReason,
IndexName: index.IndexName,
IndexVersion: segIdx.CurrentIndexVersion,
}
}
}

15
internal/datacoord/index_service.go
View File

@ -19,6 +19,7 @@ package datacoord
import (
"context"
"fmt"
"math"
"strings"
"time"
@ -709,6 +710,9 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
pendingIndexRows = int64(0)
)
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for segID, seg := range segments {
if seg.state != commonpb.SegmentState_Flushed && seg.state != commonpb.SegmentState_Flushing {
continue
@ -745,6 +749,12 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
case commonpb.IndexState_Finished:
cntFinished++
indexedRows += seg.numRows
if segIdx.IndexVersion < minIndexVersion {
minIndexVersion = segIdx.IndexVersion
}
if segIdx.IndexVersion > maxIndexVersion {
maxIndexVersion = segIdx.IndexVersion
}
case commonpb.IndexState_Failed:
cntFailed++
failReason += fmt.Sprintf("%d: %s;", segID, segIdx.FailReason)
@ -758,6 +768,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
}
indexInfo.TotalRows = totalRows
indexInfo.PendingIndexRows = pendingIndexRows
indexInfo.MinIndexVersion = minIndexVersion
indexInfo.MaxIndexVersion = maxIndexVersion
switch {
case cntFailed > 0:
indexInfo.State = commonpb.IndexState_Failed
@ -773,7 +785,8 @@ func (s *Server) completeIndexInfo(indexInfo *indexpb.IndexInfo, index *model.In
log.RatedInfo(60, "completeIndexInfo success", zap.Int64("collectionID", index.CollectionID), zap.Int64("indexID", index.IndexID),
zap.Int64("totalRows", indexInfo.TotalRows), zap.Int64("indexRows", indexInfo.IndexedRows),
zap.Int64("pendingIndexRows", indexInfo.PendingIndexRows),
zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason))
zap.String("state", indexInfo.State.String()), zap.String("failReason", indexInfo.IndexStateFailReason),
zap.Int32("minIndexVersion", indexInfo.MinIndexVersion), zap.Int32("maxIndexVersion", indexInfo.MaxIndexVersion))
}
// GetIndexBuildProgress get the index building progress by num rows.

79
internal/datacoord/index_service_test.go
View File

@ -18,6 +18,7 @@ package datacoord
import (
"context"
"math"
"testing"
"time"
@ -1472,6 +1473,7 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil,
IndexSerializedSize: 0,
WriteHandoff: false,
CurrentIndexVersion: 7,
})
segIdx1.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID,
@ -1489,6 +1491,8 @@ func TestServer_DescribeIndex(t *testing.T) {
IndexFileKeys: nil,
IndexSerializedSize: 0,
WriteHandoff: false,
// deleted index
CurrentIndexVersion: 6,
})
segIdx1.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID,
@ -1545,28 +1549,30 @@ func TestServer_DescribeIndex(t *testing.T) {
segIdx2 := typeutil.NewConcurrentMap[UniqueID, *model.SegmentIndex]()
segIdx2.Insert(indexID, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID,
BuildID: buildID,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID,
BuildID: buildID,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
segIdx2.Insert(indexID+1, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 1,
BuildID: buildID + 1,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 1,
BuildID: buildID + 1,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
segIdx2.Insert(indexID+3, &model.SegmentIndex{
SegmentID: segID - 1,
@ -1594,16 +1600,17 @@ func TestServer_DescribeIndex(t *testing.T) {
CreatedUTCTime: createTS,
})
segIdx2.Insert(indexID+5, &model.SegmentIndex{
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 5,
BuildID: buildID + 5,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
SegmentID: segID - 1,
CollectionID: collID,
PartitionID: partID,
NumRows: 10000,
IndexID: indexID + 5,
BuildID: buildID + 5,
NodeID: 0,
IndexVersion: 1,
IndexState: commonpb.IndexState_Finished,
CreatedUTCTime: createTS,
CurrentIndexVersion: 6,
})
s.meta.indexMeta.segmentIndexes.Insert(segID-1, segIdx2)
@ -1625,6 +1632,18 @@ func TestServer_DescribeIndex(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, 5, len(resp.GetIndexInfos()))
minIndexVersion := int32(math.MaxInt32)
maxIndexVersion := int32(math.MinInt32)
for _, indexInfo := range resp.GetIndexInfos() {
if indexInfo.GetMinIndexVersion() < minIndexVersion {
minIndexVersion = indexInfo.GetMinIndexVersion()
}
if indexInfo.GetMaxIndexVersion() > maxIndexVersion {
maxIndexVersion = indexInfo.GetMaxIndexVersion()
}
}
assert.Equal(t, int32(7), minIndexVersion)
assert.Equal(t, int32(7), maxIndexVersion)
})
t.Run("describe after drop index", func(t *testing.T) {

3
internal/distributed/proxy/httpserver/constant.go
View File

@ -159,6 +159,9 @@ const (
HTTPReturnIndexState = "indexState"
HTTPReturnIndexFailReason = "failReason"
HTTPReturnMinIndexVersion = "minIndexVersion"
HTTPReturnMaxIndexVersion = "maxIndexVersion"
HTTPReturnDistance = "distance"
HTTPReturnRowCount = "rowCount"

2
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -2280,6 +2280,8 @@ func (h *HandlersV2) describeIndex(ctx context.Context, c *gin.Context, anyReq a
HTTPReturnIndexIndexedRows: indexDescription.IndexedRows,
HTTPReturnIndexState: indexDescription.State.String(),
HTTPReturnIndexFailReason: indexDescription.IndexStateFailReason,
HTTPReturnMinIndexVersion: indexDescription.MinIndexVersion,
HTTPReturnMaxIndexVersion: indexDescription.MaxIndexVersion,
}
indexInfos = append(indexInfos, indexInfo)
}

4
internal/proxy/task_index.go
View File

@ -845,6 +845,8 @@ func (dit *describeIndexTask) Execute(ctx context.Context) error {
PendingIndexRows: indexInfo.GetPendingIndexRows(),
State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
}
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
}
@ -954,6 +956,8 @@ func (dit *getIndexStatisticsTask) Execute(ctx context.Context) error {
TotalRows: indexInfo.GetTotalRows(),
State: indexInfo.GetState(),
IndexStateFailReason: indexInfo.GetIndexStateFailReason(),
MinIndexVersion: indexInfo.GetMinIndexVersion(),
MaxIndexVersion: indexInfo.GetMaxIndexVersion(),
}
dit.result.IndexDescriptions = append(dit.result.IndexDescriptions, desc)
}

2
pkg/go.mod
View File

@ -20,7 +20,7 @@ require (
github.com/jolestar/go-commons-pool/v2 v2.1.2
github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.9
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad
github.com/minio/minio-go/v7 v7.0.73
github.com/panjf2000/ants/v2 v2.7.2
github.com/prometheus/client_golang v1.14.0

4
pkg/go.sum
View File

@ -559,8 +559,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030 h1:TZR8QUxLExFI/LBexz2/ucWGvvDZv73IXb/dUNlwRrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250429023443-20f3b3d3e030/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad h1:sjhse0GoiU4ktY+aKSNeWg/zBS5jXvPFx818wGAVpZs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250514091016-4080770055ad/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=

3
pkg/proto/index_coord.proto
View File

@ -52,6 +52,8 @@ message IndexInfo {
bool is_auto_index = 11;
repeated common.KeyValuePair user_index_params = 12;
int64 pending_index_rows = 13;
int32 min_index_version = 14;
int32 max_index_version = 15;
}
message FieldIndex {
@ -116,6 +118,7 @@ message SegmentIndexState {
common.IndexState state = 2;
string fail_reason = 3;
string index_name = 4;
int32 index_version = 5;
}
message GetSegmentIndexStateResponse {

1026
pkg/proto/indexpb/index_coord.pb.go
View File

File diff suppressed because it is too large Load Diff