admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Get index status based on segment generation time (#17659)

Browse Source 
Signed-off-by: Cai.Zhang <cai.zhang@zilliz.com>
This commit is contained in:
cai.zhang 2022-06-21 18:44:12 +08:00 committed by GitHub
parent e43f487f02
commit 4efdd0929c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 187 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/proto/etcd_meta.proto
View File

@ -16,6 +16,7 @@ message IndexInfo {
int64 indexID = 2; int64 indexID = 2;
repeated common.KeyValuePair index_params = 3; repeated common.KeyValuePair index_params = 3;
bool deleted = 4; bool deleted = 4;
uint64 create_time = 5;
} }
message FieldIndexInfo{ message FieldIndexInfo{
@ -46,7 +47,7 @@ message SegmentIndexInfo {
int64 indexID = 5; int64 indexID = 5;
int64 buildID = 6; int64 buildID = 6;
bool enable_index = 7; bool enable_index = 7;
bool by_auto_flush = 8; uint64 create_time = 8;
} }
message CollectionMeta { message CollectionMeta {

112
internal/proto/etcdpb/etcd_meta.pb.go
View File

@ -82,6 +82,7 @@ type IndexInfo struct {
IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"` IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,3,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"` IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,3,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"`
Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
CreateTime uint64 `protobuf:"varint,5,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"` XXX_sizecache int32 `json:"-"`
@ -140,6 +141,13 @@ func (m *IndexInfo) GetDeleted() bool {
return false return false
} }
func (m *IndexInfo) GetCreateTime() uint64 {
if m != nil {
return m.CreateTime
}
return 0
}
type FieldIndexInfo struct { type FieldIndexInfo struct {
FiledID int64 `protobuf:"varint,1,opt,name=filedID,proto3" json:"filedID,omitempty"` FiledID int64 `protobuf:"varint,1,opt,name=filedID,proto3" json:"filedID,omitempty"`
IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"` IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
@ -322,7 +330,7 @@ type SegmentIndexInfo struct {
IndexID int64 `protobuf:"varint,5,opt,name=indexID,proto3" json:"indexID,omitempty"` IndexID int64 `protobuf:"varint,5,opt,name=indexID,proto3" json:"indexID,omitempty"`
BuildID int64 `protobuf:"varint,6,opt,name=buildID,proto3" json:"buildID,omitempty"` BuildID int64 `protobuf:"varint,6,opt,name=buildID,proto3" json:"buildID,omitempty"`
EnableIndex bool `protobuf:"varint,7,opt,name=enable_index,json=enableIndex,proto3" json:"enable_index,omitempty"` EnableIndex bool `protobuf:"varint,7,opt,name=enable_index,json=enableIndex,proto3" json:"enable_index,omitempty"`
ByAutoFlush bool `protobuf:"varint,8,opt,name=by_auto_flush,json=byAutoFlush,proto3" json:"by_auto_flush,omitempty"` CreateTime uint64 `protobuf:"varint,8,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"` XXX_sizecache int32 `json:"-"`
@ -402,11 +410,11 @@ func (m *SegmentIndexInfo) GetEnableIndex() bool {
return false return false
} }
func (m *SegmentIndexInfo) GetByAutoFlush() bool { func (m *SegmentIndexInfo) GetCreateTime() uint64 {
if m != nil { if m != nil {
return m.ByAutoFlush return m.CreateTime
} }
return false return 0
} }
type CollectionMeta struct { type CollectionMeta struct {
@ -500,52 +508,52 @@ func init() {
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) } func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{ var fileDescriptor_975d306d62b73e88 = []byte{
// 751 bytes of a gzipped FileDescriptorProto // 737 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6b, 0xeb, 0x46, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4b, 0x6b, 0xeb, 0x46,
0x14, 0x45, 0x96, 0x63, 0x47, 0xd7, 0x8a, 0x5f, 0x32, 0xfd, 0x60, 0x08, 0x69, 0xab, 0x27, 0x78, 0x14, 0x46, 0x96, 0x1f, 0xd1, 0xb1, 0xae, 0x6f, 0x32, 0x7d, 0x30, 0x84, 0xb4, 0xd5, 0x35, 0xdc,
0x45, 0x50, 0x6a, 0xd3, 0xbc, 0xd2, 0x5d, 0xa1, 0xaf, 0x16, 0x01, 0xd3, 0xf6, 0x61, 0x26, 0xa1, 0x22, 0x28, 0xb5, 0x69, 0x6e, 0xe9, 0xae, 0xd0, 0xd6, 0x22, 0x60, 0xda, 0x06, 0x33, 0x09, 0x5d,
0x8b, 0x6e, 0xc4, 0x58, 0xba, 0xb6, 0x07, 0xf4, 0x61, 0x34, 0xa3, 0x10, 0xef, 0xba, 0xef, 0xdf, 0x74, 0x23, 0xc6, 0xd2, 0xb1, 0x3d, 0xa0, 0x87, 0xd1, 0x8c, 0x42, 0xbc, 0xeb, 0xb2, 0x7f, 0xac,
0xe8, 0xbf, 0xeb, 0x7f, 0x28, 0x45, 0x33, 0x92, 0xfc, 0x91, 0x64, 0xd9, 0x9d, 0xcf, 0x99, 0x7b, 0xbf, 0xa6, 0xff, 0xa1, 0x14, 0xcd, 0x48, 0xf2, 0x2b, 0x59, 0xde, 0x9d, 0xbf, 0xef, 0x3c, 0x34,
0xaf, 0xe6, 0x9e, 0x39, 0xc7, 0xf0, 0x06, 0x55, 0x9c, 0x44, 0x19, 0x2a, 0x3e, 0xd9, 0x96, 0x85, 0xe7, 0x9c, 0xef, 0x33, 0xbc, 0x45, 0x15, 0xc5, 0x61, 0x8a, 0x8a, 0x4f, 0xb6, 0x45, 0xae, 0x72,
0x2a, 0xc8, 0x55, 0x26, 0xd2, 0xc7, 0x4a, 0x1a, 0x34, 0xa9, 0x4f, 0xaf, 0xdd, 0xb8, 0xc8, 0xb2, 0x72, 0x95, 0x8a, 0xe4, 0xa9, 0x94, 0x06, 0x4d, 0xaa, 0xe8, 0xb5, 0x1b, 0xe5, 0x69, 0x9a, 0x67,
0x22, 0x37, 0xd4, 0xb5, 0x2b, 0xe3, 0x0d, 0x66, 0x4d, 0xb9, 0xff, 0xa7, 0x05, 0xce, 0xa2, 0x2c, 0x86, 0xba, 0x76, 0x65, 0xb4, 0xc1, 0xb4, 0x4e, 0x1f, 0xff, 0x65, 0x81, 0xb3, 0x28, 0xf2, 0xe7,
0x9e, 0x76, 0xbf, 0xa1, 0xe2, 0x64, 0x0c, 0xbd, 0x79, 0x48, 0x2d, 0xcf, 0x0a, 0x6c, 0xd6, 0x9b, 0xdd, 0xef, 0xa8, 0x38, 0x19, 0x41, 0x67, 0x1e, 0x50, 0xcb, 0xb3, 0x7c, 0x9b, 0x75, 0xe6, 0x01,
0x87, 0xe4, 0x07, 0x18, 0xf2, 0x24, 0x29, 0x51, 0x4a, 0xda, 0xf3, 0xac, 0x60, 0x74, 0x7b, 0x33, 0xf9, 0x01, 0x06, 0x3c, 0x8e, 0x0b, 0x94, 0x92, 0x76, 0x3c, 0xcb, 0x1f, 0xde, 0xde, 0x4c, 0x8e,
0x39, 0x1a, 0xdf, 0x0c, 0xfe, 0x60, 0x6a, 0x58, 0x5b, 0x4c, 0xbe, 0x81, 0xab, 0x12, 0x65, 0x95, 0xda, 0xd7, 0x8d, 0x7f, 0x36, 0x39, 0xac, 0x49, 0x26, 0xdf, 0xc0, 0x55, 0x81, 0xb2, 0x4c, 0x54,
0xaa, 0x28, 0xde, 0xf0, 0x3c, 0xc7, 0x74, 0x1e, 0x4a, 0x6a, 0x7b, 0x76, 0xe0, 0xb0, 0x4b, 0x73, 0x18, 0x6d, 0x78, 0x96, 0x61, 0x32, 0x0f, 0x24, 0xb5, 0x3d, 0xdb, 0x77, 0xd8, 0xa5, 0x09, 0xcc,
0x30, 0xeb, 0x78, 0xff, 0x6f, 0x0b, 0x9c, 0x79, 0x9e, 0xe0, 0xd3, 0x3c, 0x5f, 0x15, 0xe4, 0x0b, 0x5a, 0x7e, 0xfc, 0x8f, 0x05, 0xce, 0x3c, 0x8b, 0xf1, 0x79, 0x9e, 0xad, 0x72, 0xf2, 0x05, 0x80,
0x00, 0x51, 0x83, 0x28, 0xe7, 0x19, 0xea, 0xab, 0x38, 0xcc, 0xd1, 0xcc, 0x47, 0x9e, 0x21, 0xa1, 0xa8, 0x40, 0x98, 0xf1, 0x14, 0xf5, 0x53, 0x1c, 0xe6, 0x68, 0xe6, 0x9e, 0xa7, 0x48, 0x28, 0x0c,
0x30, 0xd4, 0x60, 0x1e, 0xea, 0x1b, 0xd9, 0xac, 0x85, 0x24, 0x04, 0xd7, 0x34, 0x6e, 0x79, 0xc9, 0x34, 0x98, 0x07, 0xfa, 0x45, 0x36, 0x6b, 0x20, 0x09, 0xc0, 0x35, 0x85, 0x5b, 0x5e, 0xf0, 0xd4,
0x33, 0xf3, 0xb9, 0xd1, 0xed, 0xdb, 0x17, 0x2f, 0xfc, 0x0b, 0xee, 0x7e, 0xe7, 0x69, 0x85, 0x0b, 0x7c, 0x6e, 0x78, 0xfb, 0xee, 0xc5, 0x07, 0xff, 0x8a, 0xbb, 0x3f, 0x78, 0x52, 0xe2, 0x82, 0x8b,
0x2e, 0x4a, 0x36, 0xd2, 0x6d, 0x0b, 0xdd, 0x55, 0xcf, 0x4f, 0x30, 0x45, 0x85, 0x09, 0xed, 0x7b, 0x82, 0x0d, 0x75, 0xd9, 0x42, 0x57, 0x55, 0xfd, 0x63, 0x4c, 0x50, 0x61, 0x4c, 0xbb, 0x9e, 0xe5,
0x56, 0x70, 0xce, 0x5a, 0xe8, 0x87, 0x30, 0xbe, 0x13, 0x98, 0x26, 0xfb, 0xab, 0x52, 0x18, 0xae, 0x5f, 0xb0, 0x06, 0x92, 0xaf, 0x60, 0x18, 0x15, 0xc8, 0x15, 0x86, 0x4a, 0xa4, 0x48, 0x7b, 0x9e,
0x44, 0x8a, 0x49, 0x27, 0x59, 0x0b, 0x5f, 0xbf, 0xa5, 0xff, 0x6f, 0x1f, 0xc6, 0xb3, 0x22, 0x4d, 0xe5, 0x77, 0x19, 0x18, 0xea, 0x51, 0xa4, 0x38, 0x0e, 0x60, 0x74, 0x27, 0x30, 0x89, 0xf7, 0xb3,
0x31, 0x56, 0xa2, 0xc8, 0xf5, 0x98, 0x53, 0xd1, 0x7f, 0x84, 0x81, 0x79, 0xa2, 0x46, 0xf3, 0x77, 0x50, 0x18, 0xac, 0x44, 0x82, 0x71, 0xbb, 0xd3, 0x06, 0xbe, 0x3e, 0xc6, 0xf8, 0xbf, 0x2e, 0x8c,
0xc7, 0x2b, 0x34, 0xcf, 0xb7, 0x1f, 0x72, 0xaf, 0x09, 0xd6, 0x34, 0x91, 0xaf, 0x60, 0x14, 0x97, 0x66, 0x79, 0x92, 0x60, 0xa4, 0x44, 0x9e, 0xe9, 0x36, 0xa7, 0x57, 0xf9, 0x11, 0xfa, 0xe6, 0x86,
0xc8, 0x15, 0x46, 0x4a, 0x64, 0x48, 0x6d, 0xcf, 0x0a, 0xfa, 0x0c, 0x0c, 0xf5, 0x20, 0x32, 0x24, 0xf5, 0x51, 0xde, 0x1f, 0xcf, 0x58, 0xdf, 0x77, 0xdf, 0xe4, 0x41, 0x13, 0xac, 0x2e, 0x3a, 0x1d,
0x3e, 0xb8, 0x5b, 0x5e, 0x2a, 0xa1, 0x2f, 0x10, 0x4a, 0xda, 0xf7, 0xec, 0xc0, 0x66, 0x47, 0x1c, 0xc4, 0x3e, 0x1d, 0x84, 0x8c, 0xc1, 0xdd, 0xf2, 0x42, 0x09, 0xfd, 0x80, 0x40, 0xd2, 0xae, 0x67,
0xf9, 0x1a, 0xc6, 0x1d, 0xae, 0x75, 0x97, 0xf4, 0x4c, 0xbf, 0xde, 0x09, 0x4b, 0xee, 0xe0, 0x62, 0xfb, 0x36, 0x3b, 0xe2, 0xc8, 0xd7, 0x30, 0x6a, 0x71, 0x75, 0x18, 0x49, 0x7b, 0xfa, 0xbc, 0x27,
0x55, 0x8b, 0x12, 0xe9, 0xfd, 0x50, 0xd2, 0xc1, 0x4b, 0xaa, 0xd7, 0x2e, 0x9c, 0x1c, 0x8b, 0xc7, 0x2c, 0xb9, 0x83, 0x37, 0xab, 0x6a, 0x29, 0xa1, 0x9e, 0x0f, 0x25, 0xed, 0xbf, 0x74, 0x96, 0x4a,
0xdc, 0x55, 0x87, 0x51, 0x92, 0x5b, 0xf8, 0xec, 0x51, 0x94, 0xaa, 0xe2, 0x69, 0xeb, 0x18, 0xfd, 0xa6, 0x93, 0xe3, 0xe5, 0x31, 0x77, 0xd5, 0x62, 0x94, 0xe4, 0x16, 0x3e, 0x7b, 0x12, 0x85, 0x2a,
0xfe, 0x92, 0x0e, 0xf5, 0x67, 0x3f, 0x69, 0x0e, 0x1b, 0xd7, 0x98, 0x6f, 0x7f, 0x0f, 0x9f, 0x6f, 0x79, 0xd2, 0x48, 0x4a, 0x0b, 0x44, 0xd2, 0x81, 0xfe, 0xec, 0x27, 0x75, 0xb0, 0x96, 0x95, 0xf9,
0x37, 0x3b, 0x29, 0xe2, 0x67, 0x4d, 0xe7, 0xba, 0xe9, 0xd3, 0xf6, 0xf4, 0xa8, 0xeb, 0x27, 0xb8, 0xf6, 0xf7, 0xf0, 0xf9, 0x76, 0xb3, 0x93, 0x22, 0x3a, 0x2b, 0xba, 0xd0, 0x45, 0x9f, 0x36, 0xd1,
0xe9, 0x76, 0x88, 0x8c, 0x2a, 0x89, 0x56, 0x4a, 0x2a, 0x9e, 0x6d, 0x25, 0x75, 0x3c, 0x3b, 0xe8, 0xa3, 0xaa, 0x9f, 0xe0, 0xa6, 0x9d, 0x21, 0x34, 0x5b, 0x89, 0xf5, 0xa6, 0xa4, 0xe2, 0xe9, 0x56,
0xb3, 0xeb, 0xae, 0x66, 0x66, 0x4a, 0x1e, 0xba, 0x8a, 0xda, 0xa1, 0x72, 0xc3, 0xcb, 0x44, 0x46, 0x52, 0xc7, 0xb3, 0xfd, 0x2e, 0xbb, 0x6e, 0x73, 0x66, 0x26, 0xe5, 0xb1, 0xcd, 0xa8, 0x24, 0x2c,
0x79, 0x95, 0x51, 0xf0, 0xac, 0xe0, 0x8c, 0x39, 0x86, 0xf9, 0x58, 0x65, 0x64, 0x0e, 0x6f, 0xa4, 0x37, 0xbc, 0x88, 0x65, 0x98, 0x95, 0x29, 0x05, 0xcf, 0xf2, 0x7b, 0xcc, 0x31, 0xcc, 0x7d, 0x99,
0xe2, 0xa5, 0x8a, 0xb6, 0x85, 0xd4, 0x13, 0x24, 0x1d, 0x69, 0x51, 0xbc, 0xd7, 0xac, 0x18, 0x72, 0x92, 0x39, 0xbc, 0x95, 0x8a, 0x17, 0x2a, 0xdc, 0xe6, 0x52, 0x77, 0x90, 0x74, 0xa8, 0x97, 0xe2,
0xc5, 0xb5, 0x13, 0xc7, 0xba, 0x71, 0xd1, 0xf6, 0x11, 0x06, 0x57, 0x71, 0x91, 0x4b, 0x21, 0x15, 0xbd, 0xa6, 0xd5, 0x80, 0x2b, 0xae, 0xa5, 0x3a, 0xd2, 0x85, 0x8b, 0xa6, 0x8e, 0x30, 0xb8, 0x8a,
0xe6, 0xf1, 0x2e, 0x4a, 0xf1, 0x11, 0x53, 0xea, 0x7a, 0x56, 0x30, 0x3e, 0x35, 0x45, 0x33, 0x6c, 0xf2, 0x4c, 0x0a, 0xa9, 0x30, 0x8b, 0x76, 0x61, 0x82, 0x4f, 0x98, 0x50, 0xd7, 0xb3, 0xfc, 0xd1,
0xb6, 0xaf, 0xfe, 0xb5, 0x2e, 0x66, 0x97, 0xf1, 0x09, 0xe3, 0xff, 0xd5, 0x83, 0xcb, 0x7b, 0x5c, 0xa9, 0x28, 0xea, 0x66, 0xb3, 0x7d, 0xf6, 0x6f, 0x55, 0x32, 0xbb, 0x8c, 0x4e, 0x98, 0xf1, 0xdf,
0x67, 0x98, 0xab, 0xbd, 0x93, 0x7d, 0x70, 0xe3, 0xbd, 0x29, 0x5b, 0x33, 0x1e, 0x71, 0xc4, 0x83, 0x1d, 0xb8, 0x7c, 0xc0, 0x75, 0x8a, 0x99, 0xda, 0x2b, 0x79, 0x0c, 0x6e, 0xb4, 0x17, 0x65, 0x23,
0xd1, 0x81, 0x45, 0x1a, 0x5f, 0x1f, 0x52, 0xe4, 0x06, 0x1c, 0xd9, 0x4c, 0x0e, 0xb5, 0xef, 0x6c, 0xc6, 0x23, 0x8e, 0x78, 0x30, 0x3c, 0x90, 0x48, 0xad, 0xeb, 0x43, 0x8a, 0xdc, 0x80, 0x23, 0xeb,
0xb6, 0x27, 0x4c, 0x5a, 0xea, 0x27, 0x0f, 0x75, 0xb2, 0x74, 0x5a, 0x34, 0x3c, 0x4c, 0xcb, 0xd9, 0xce, 0x81, 0xd6, 0x9d, 0xcd, 0xf6, 0x84, 0x71, 0x4b, 0x75, 0xf2, 0x40, 0x5b, 0x4f, 0xbb, 0x45,
0x71, 0xa6, 0x29, 0x0c, 0x97, 0x95, 0xd0, 0x3d, 0x03, 0x73, 0xd2, 0x40, 0xf2, 0x16, 0x5c, 0xcc, 0xc3, 0x43, 0xb7, 0xf4, 0x8e, 0x4d, 0x4f, 0x61, 0xb0, 0x2c, 0x85, 0xae, 0xe9, 0x9b, 0x48, 0x0d,
0xf9, 0x32, 0x45, 0xe3, 0x3c, 0x3a, 0xd4, 0x61, 0x1d, 0x19, 0x4e, 0x2f, 0x46, 0x7c, 0xb8, 0x58, 0xc9, 0x3b, 0x70, 0x31, 0xe3, 0xcb, 0x04, 0x8d, 0xf2, 0xe8, 0x40, 0xbb, 0x79, 0x68, 0x38, 0x3d,
0xee, 0x22, 0x5e, 0xa9, 0x22, 0x5a, 0xa5, 0x95, 0xdc, 0xd0, 0x73, 0x53, 0xb3, 0xdc, 0x7d, 0xa8, 0xd8, 0xa9, 0x11, 0x2e, 0xce, 0x1c, 0xfd, 0xaf, 0x75, 0xe8, 0xc5, 0x17, 0xff, 0x21, 0x3f, 0xb6,
0x54, 0x71, 0x57, 0x53, 0xfe, 0x3f, 0xd6, 0x61, 0x1c, 0x5f, 0xfc, 0x0f, 0xfc, 0xbf, 0xe3, 0xf8, 0x17, 0xbf, 0x04, 0x68, 0x37, 0xd4, 0x38, 0xf1, 0x80, 0x21, 0xef, 0x0f, 0x7c, 0x18, 0x2a, 0xbe,
0x25, 0x40, 0x27, 0x52, 0x1b, 0xc6, 0x03, 0x86, 0xbc, 0x3b, 0x88, 0x62, 0xa4, 0xf8, 0xba, 0x8d, 0x6e, 0x7c, 0xf8, 0xa6, 0x65, 0x1f, 0xf9, 0x5a, 0x9e, 0x59, 0xba, 0x7f, 0x6e, 0xe9, 0x5f, 0x3e,
0xe2, 0x45, 0xc7, 0x3e, 0xf0, 0xb5, 0x7c, 0x96, 0xea, 0xc1, 0xf3, 0x54, 0xff, 0xfc, 0xfe, 0x8f, 0xfc, 0xf9, 0xdd, 0x5a, 0xa8, 0x4d, 0xb9, 0xac, 0xc4, 0x32, 0x35, 0x63, 0x7c, 0x2b, 0xf2, 0xfa,
0xef, 0xd6, 0x42, 0x6d, 0xaa, 0x65, 0xed, 0x97, 0xa9, 0x59, 0xe3, 0x5b, 0x51, 0x34, 0xbf, 0xa6, 0xd7, 0x54, 0x64, 0x0a, 0x8b, 0x8c, 0x27, 0x53, 0x3d, 0xd9, 0xb4, 0xb2, 0xec, 0x76, 0xb9, 0xec,
0x22, 0x57, 0x58, 0xe6, 0x3c, 0x9d, 0xea, 0xcd, 0xa6, 0x75, 0x6a, 0xb7, 0xcb, 0xe5, 0x40, 0xa3, 0x6b, 0xf4, 0xe1, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8c, 0x2d, 0xaa, 0x13, 0x88, 0x06, 0x00,
0xf7, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x19, 0xa3, 0x49, 0xe3, 0x6a, 0x06, 0x00, 0x00, 0x00,
} }

33
internal/rootcoord/meta_table.go
View File

@ -953,10 +953,12 @@ func (mt *MetaTable) GetInitBuildIDs(collName, indexName string) ([]UniqueID, er
} }
var indexID typeutil.UniqueID var indexID typeutil.UniqueID
var indexIDCreateTS uint64
for _, info := range collMeta.FieldIndexes { for _, info := range collMeta.FieldIndexes {
idxMeta, ok := mt.indexID2Meta[info.IndexID] idxMeta, ok := mt.indexID2Meta[info.IndexID]
if ok && idxMeta.IndexName == indexName { if ok && idxMeta.IndexName == indexName {
indexID = info.IndexID indexID = info.IndexID
indexIDCreateTS = idxMeta.CreateTime
break break
} }
} }
@ -970,7 +972,7 @@ func (mt *MetaTable) GetInitBuildIDs(collName, indexName string) ([]UniqueID, er
initBuildIDs := make([]UniqueID, 0) initBuildIDs := make([]UniqueID, 0)
for _, indexID2Info := range mt.segID2IndexMeta { for _, indexID2Info := range mt.segID2IndexMeta {
segIndexInfo, ok := indexID2Info[indexID] segIndexInfo, ok := indexID2Info[indexID]
if ok && segIndexInfo.EnableIndex && !segIndexInfo.ByAutoFlush { if ok && segIndexInfo.EnableIndex && segIndexInfo.CreateTime <= indexIDCreateTS {
initBuildIDs = append(initBuildIDs, segIndexInfo.BuildID) initBuildIDs = append(initBuildIDs, segIndexInfo.BuildID)
} }
} }
@ -1282,21 +1284,21 @@ func (mt *MetaTable) checkFieldCanBeIndexed(collMeta pb.CollectionInfo, fieldSch
return nil return nil
} }
func (mt *MetaTable) checkFieldIndexDuplicate(collMeta pb.CollectionInfo, fieldSchema schemapb.FieldSchema, idxInfo *pb.IndexInfo) (duplicate bool, err error) { func (mt *MetaTable) checkFieldIndexDuplicate(collMeta pb.CollectionInfo, fieldSchema schemapb.FieldSchema, idxInfo *pb.IndexInfo) (duplicate bool, dupIdxInfo *pb.IndexInfo, err error) {
for _, f := range collMeta.FieldIndexes { for _, f := range collMeta.FieldIndexes {
if info, ok := mt.indexID2Meta[f.IndexID]; ok && !info.GetDeleted() { if info, ok := mt.indexID2Meta[f.IndexID]; ok && !info.GetDeleted() {
if info.IndexName == idxInfo.IndexName { if info.IndexName == idxInfo.IndexName {
// the index name must be different for different indexes // the index name must be different for different indexes
if f.FiledID != fieldSchema.FieldID || !EqualKeyPairArray(info.IndexParams, idxInfo.IndexParams) { if f.FiledID != fieldSchema.FieldID || !EqualKeyPairArray(info.IndexParams, idxInfo.IndexParams) {
return false, fmt.Errorf("index already exists, collection: %s, field: %s, index: %s", collMeta.GetSchema().GetName(), fieldSchema.GetName(), idxInfo.GetIndexName()) return false, nil, fmt.Errorf("index already exists, collection: %s, field: %s, index: %s", collMeta.GetSchema().GetName(), fieldSchema.GetName(), idxInfo.GetIndexName())
} }
// same index name, index params, and fieldId // same index name, index params, and fieldId
return true, nil return true, proto.Clone(&info).(*pb.IndexInfo), nil
} }
} }
} }
return false, nil return false, nil, nil
} }
// GetNotIndexedSegments return segment ids which have no index // GetNotIndexedSegments return segment ids which have no index
@ -1335,7 +1337,7 @@ func (mt *MetaTable) GetNotIndexedSegments(collName string, fieldName string, id
return nil, schemapb.FieldSchema{}, err return nil, schemapb.FieldSchema{}, err
} }
dupIdx, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo) dupIdx, dupIdxInfo, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
if err != nil { if err != nil {
// error here if index already exists. // error here if index already exists.
return nil, fieldSchema, err return nil, fieldSchema, err
@ -1374,6 +1376,25 @@ func (mt *MetaTable) GetNotIndexedSegments(collName string, fieldName string, id
mt.collID2Meta[collMeta.ID] = collMeta mt.collID2Meta[collMeta.ID] = collMeta
mt.indexID2Meta[idx.IndexID] = *idxInfo mt.indexID2Meta[idx.IndexID] = *idxInfo
} else {
log.Info("index has been created, update timestamp for IndexID", zap.Int64("indexID", dupIdxInfo.IndexID))
// just update create time for IndexID
dupIdxInfo.CreateTime = idxInfo.CreateTime
k := fmt.Sprintf("%s/%d/%d", IndexMetaPrefix, collMeta.ID, dupIdxInfo.IndexID)
v, err := proto.Marshal(dupIdxInfo)
if err != nil {
log.Error("MetaTable GetNotIndexedSegments Marshal idxInfo fail",
zap.String("key", k), zap.Error(err))
return nil, schemapb.FieldSchema{}, fmt.Errorf("metaTable GetNotIndexedSegments Marshal idxInfo fail key:%s, err:%w", k, err)
}
meta := map[string]string{k: string(v)}
err = mt.txn.MultiSave(meta)
if err != nil {
log.Error("TxnKV MultiSave fail", zap.Error(err))
panic("TxnKV MultiSave fail")
}
mt.indexID2Meta[dupIdxInfo.IndexID] = *dupIdxInfo
} }
rstID := make([]typeutil.UniqueID, 0, 16) rstID := make([]typeutil.UniqueID, 0, 16)

83
internal/rootcoord/meta_table_test.go
View File

@ -1367,7 +1367,7 @@ func TestMetaTable_checkFieldIndexDuplicate(t *testing.T) {
} }
fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 101} fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 101}
idxInfo := &pb.IndexInfo{IndexName: "test"} idxInfo := &pb.IndexInfo{IndexName: "test"}
_, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo) _, _, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -1384,7 +1384,7 @@ func TestMetaTable_checkFieldIndexDuplicate(t *testing.T) {
} }
fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 100} fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 100}
idxInfo := &pb.IndexInfo{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "not_Value"}}} idxInfo := &pb.IndexInfo{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "not_Value"}}}
_, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo) _, _, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -1401,9 +1401,10 @@ func TestMetaTable_checkFieldIndexDuplicate(t *testing.T) {
} }
fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 100} fieldSchema := schemapb.FieldSchema{Name: "test", FieldID: 100}
idxInfo := &pb.IndexInfo{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "Value"}}} idxInfo := &pb.IndexInfo{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "Value"}}}
duplicate, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo) duplicate, dupIdxInfo, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, duplicate) assert.True(t, duplicate)
assert.Equal(t, idxInfo.IndexName, dupIdxInfo.IndexName)
}) })
t.Run("field not found", func(t *testing.T) { t.Run("field not found", func(t *testing.T) {
@ -1415,9 +1416,10 @@ func TestMetaTable_checkFieldIndexDuplicate(t *testing.T) {
FieldID: 101, FieldID: 101,
} }
idxInfo := &pb.IndexInfo{} idxInfo := &pb.IndexInfo{}
duplicate, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo) duplicate, dupIdxInfo, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, duplicate) assert.False(t, duplicate)
assert.Nil(t, dupIdxInfo)
}) })
} }
@ -1452,10 +1454,12 @@ func TestMetaTable_GetInitBuildIDs(t *testing.T) {
1: { 1: {
IndexName: "GetInitBuildID-Index-1", IndexName: "GetInitBuildID-Index-1",
IndexID: 1, IndexID: 1,
CreateTime: 10,
}, },
2: { 2: {
IndexName: "GetInitBuildID-Index-2", IndexName: "GetInitBuildID-Index-2",
IndexID: 2, IndexID: 2,
CreateTime: 10,
}, },
}, },
} }
@ -1476,6 +1480,7 @@ func TestMetaTable_GetInitBuildIDs(t *testing.T) {
mt.indexID2Meta[3] = pb.IndexInfo{ mt.indexID2Meta[3] = pb.IndexInfo{
IndexName: indexName, IndexName: indexName,
IndexID: 3, IndexID: 3,
CreateTime: 10,
} }
mt.segID2IndexMeta = map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo{ mt.segID2IndexMeta = map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo{
@ -1483,13 +1488,14 @@ func TestMetaTable_GetInitBuildIDs(t *testing.T) {
1: { 1: {
IndexID: 1, IndexID: 1,
EnableIndex: true, EnableIndex: true,
CreateTime: 5,
}, },
}, },
5: { 5: {
3: { 3: {
IndexID: 3, IndexID: 3,
EnableIndex: true, EnableIndex: true,
ByAutoFlush: false, CreateTime: 5,
}, },
}, },
} }
@ -1723,3 +1729,70 @@ func TestMetaTable_MarkIndexDeleted(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
}) })
} }
func TestMetaTable_GetNotIndexedSegments(t *testing.T) {
var (
collName = "MarkIndexDeleted-Coll"
fieldName = "MarkIndexDeleted-Field"
indexName = "MarkIndexDeleted-Index"
collID = UniqueID(1)
fieldID = UniqueID(100)
indexID = UniqueID(1000)
segID = UniqueID(10000)
)
mt := &MetaTable{
txn: &mockTestTxnKV{
multiSave: func(kvs map[string]string) error {
return nil
},
multiRemove: func(keys []string) error {
return nil
},
save: func(key, value string) error {
return nil
},
},
collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{
collID: {
FieldIndexes: []*pb.FieldIndexInfo{
{
FiledID: fieldID,
IndexID: indexID,
},
},
Schema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
FieldID: fieldID,
Name: fieldName,
DataType: schemapb.DataType_FloatVector,
IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
},
},
},
},
},
collName2ID: map[string]typeutil.UniqueID{
collName: collID,
},
indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{
indexID: {
IndexName: indexName,
IndexID: indexID,
IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
Deleted: false,
},
},
}
t.Run("GetNotIndexedSegments", func(t *testing.T) {
idxInfo := &pb.IndexInfo{
IndexName: indexName,
IndexID: indexID,
IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
CreateTime: 0,
}
_, _, err := mt.GetNotIndexedSegments(collName, fieldName, idxInfo, []UniqueID{segID})
assert.NoError(t, err)
})
}

11
internal/rootcoord/root_coord.go
View File

@ -426,14 +426,19 @@ func (c *Core) createIndexForSegment(ctx context.Context, collID, partID, segID
field, err := GetFieldSchemaByID(&collMeta, fieldIndex.FiledID) field, err := GetFieldSchemaByID(&collMeta, fieldIndex.FiledID)
if err != nil { if err != nil {
log.Debug("GetFieldSchemaByID failed", log.Error("GetFieldSchemaByID failed",
zap.Int64("collectionID", collID), zap.Int64("collectionID", collID),
zap.Int64("fieldID", fieldIndex.FiledID)) zap.Int64("fieldID", fieldIndex.FiledID), zap.Error(err))
return err return err
} }
if c.MetaTable.IsSegmentIndexed(segID, field, indexMeta.IndexParams) { if c.MetaTable.IsSegmentIndexed(segID, field, indexMeta.IndexParams) {
continue continue
} }
createTS, err := c.TSOAllocator(1)
if err != nil {
log.Error("RootCoord alloc timestamp failed", zap.Int64("collectionID", collID), zap.Error(err))
return err
}
segIndexInfo := etcdpb.SegmentIndexInfo{ segIndexInfo := etcdpb.SegmentIndexInfo{
CollectionID: collMeta.ID, CollectionID: collMeta.ID,
@ -442,7 +447,7 @@ func (c *Core) createIndexForSegment(ctx context.Context, collID, partID, segID
FieldID: fieldIndex.FiledID, FieldID: fieldIndex.FiledID,
IndexID: fieldIndex.IndexID, IndexID: fieldIndex.IndexID,
EnableIndex: false, EnableIndex: false,
ByAutoFlush: true, CreateTime: createTS,
} }
buildID, err := c.BuildIndex(ctx, segID, numRows, binlogs, field, &indexMeta, false) buildID, err := c.BuildIndex(ctx, segID, numRows, binlogs, field, &indexMeta, false)
if err != nil { if err != nil {

7
internal/rootcoord/task.go
View File

@ -958,10 +958,15 @@ func (t *CreateIndexReqTask) Execute(ctx context.Context) error {
if err != nil { if err != nil {
return err return err
} }
createTS, err := t.core.TSOAllocator(1)
if err != nil {
return err
}
idxInfo := &etcdpb.IndexInfo{ idxInfo := &etcdpb.IndexInfo{
IndexName: indexName, IndexName: indexName,
IndexID: indexID, IndexID: indexID,
IndexParams: t.Req.ExtraParams, IndexParams: t.Req.ExtraParams,
CreateTime: createTS,
} }
log.Info("create index for collection", log.Info("create index for collection",
zap.String("collection", t.Req.GetCollectionName()), zap.String("collection", t.Req.GetCollectionName()),
@ -1000,7 +1005,7 @@ func (t *CreateIndexReqTask) Execute(ctx context.Context) error {
FieldID: field.FieldID, FieldID: field.FieldID,
IndexID: idxInfo.IndexID, IndexID: idxInfo.IndexID,
EnableIndex: false, EnableIndex: false,
ByAutoFlush: false, CreateTime: createTS,
} }
info.BuildID, err = t.core.BuildIndex(ctx, segID, segID2Binlog[segID].GetNumOfRows(), segID2Binlog[segID].GetFieldBinlogs(), &field, idxInfo, false) info.BuildID, err = t.core.BuildIndex(ctx, segID, segID2Binlog[segID].GetNumOfRows(), segID2Binlog[segID].GetFieldBinlogs(), &field, idxInfo, false)
if err != nil { if err != nil {