admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Compatible with scalar index types marisa-trie and Ascending (#27638)

Browse Source 
Signed-off-by: xige-16 <xi.ge@zilliz.com>
This commit is contained in:
xige-16 2023-10-15 13:52:06 +08:00 committed by GitHub
parent c5ea31316d
commit 6cbb67832f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 135 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
internal/proxy/task_index.go
View File

@ -140,15 +140,21 @@ func (cit *createIndexTask) parseIndexParams() error {
if !isVecIndex {
specifyIndexType, exist := indexParamsMap[common.IndexTypeKey]
if cit.fieldSchema.DataType == schemapb.DataType_VarChar {
if exist && specifyIndexType != DefaultStringIndexType {
if !exist {
indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType
}
if exist && !validateStringIndexType(specifyIndexType) {
return merr.WrapErrParameterInvalid(DefaultStringIndexType, specifyIndexType, "index type not match")
}
indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType
} else if typeutil.IsArithmetic(cit.fieldSchema.DataType) {
if exist && specifyIndexType != DefaultIndexType {
return merr.WrapErrParameterInvalid(DefaultIndexType, specifyIndexType, "index type not match")
if !exist {
indexParamsMap[common.IndexTypeKey] = DefaultArithmeticIndexType
}
if exist && !validateArithmeticIndexType(specifyIndexType) {
return merr.WrapErrParameterInvalid(DefaultArithmeticIndexType, specifyIndexType, "index type not match")
}
indexParamsMap[common.IndexTypeKey] = DefaultIndexType
} else {
return merr.WrapErrParameterInvalid("supported field",
fmt.Sprintf("create index on %s field", cit.fieldSchema.DataType.String()),

112
internal/proxy/task_index_test.go
View File

@ -498,6 +498,118 @@ func Test_parseIndexParams(t *testing.T) {
assert.Error(t, err)
})
t.Run("create index on VarChar field", func(t *testing.T) {
cit := &createIndexTask{
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: DefaultStringIndexType,
},
},
IndexName: "",
},
fieldSchema: &schemapb.FieldSchema{
FieldID: 101,
Name: "FieldID",
IsPrimaryKey: false,
DataType: schemapb.DataType_VarChar,
},
}
err := cit.parseIndexParams()
assert.NoError(t, err)
})
t.Run("create index on Arithmetic field", func(t *testing.T) {
cit := &createIndexTask{
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: DefaultArithmeticIndexType,
},
},
IndexName: "",
},
fieldSchema: &schemapb.FieldSchema{
FieldID: 101,
Name: "FieldID",
IsPrimaryKey: false,
DataType: schemapb.DataType_Int64,
},
}
err := cit.parseIndexParams()
assert.NoError(t, err)
})
// Compatible with the old version <= 2.3.0
t.Run("create marisa-trie index on VarChar field", func(t *testing.T) {
cit := &createIndexTask{
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "marisa-trie",
},
},
IndexName: "",
},
fieldSchema: &schemapb.FieldSchema{
FieldID: 101,
Name: "FieldID",
IsPrimaryKey: false,
DataType: schemapb.DataType_VarChar,
},
}
err := cit.parseIndexParams()
assert.NoError(t, err)
})
// Compatible with the old version <= 2.3.0
t.Run("create Asceneding index on Arithmetic field", func(t *testing.T) {
cit := &createIndexTask{
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "Asceneding",
},
},
IndexName: "",
},
fieldSchema: &schemapb.FieldSchema{
FieldID: 101,
Name: "FieldID",
IsPrimaryKey: false,
DataType: schemapb.DataType_Int64,
},
}
err := cit.parseIndexParams()
assert.NoError(t, err)
})
t.Run("create unsupported index on Arithmetic field", func(t *testing.T) {
cit := &createIndexTask{
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "invalid_type",
},
},
IndexName: "",
},
fieldSchema: &schemapb.FieldSchema{
FieldID: 101,
Name: "FieldID",
IsPrimaryKey: false,
DataType: schemapb.DataType_Int64,
},
}
err := cit.parseIndexParams()
assert.Error(t, err)
})
t.Run("create index on array field", func(t *testing.T) {
cit3 := &createIndexTask{
Condition: nil,

14
internal/proxy/util.go
View File

@ -60,8 +60,8 @@ const (
defaultMaxArrayCapacity = 4096
// DefaultIndexType name of default index type for scalar field
DefaultIndexType = "STL_SORT"
// DefaultArithmeticIndexType name of default index type for scalar field
DefaultArithmeticIndexType = "STL_SORT"
// DefaultStringIndexType name of default index type for varChar/string field
DefaultStringIndexType = "Trie"
@ -244,6 +244,16 @@ func validatePartitionTag(partitionTag string, strictCheck bool) error {
return nil
}
func validateStringIndexType(indexType string) bool {
// compatible with the index type marisa-trie of attu versions prior to 2.3.0
return indexType == DefaultStringIndexType || indexType == "marisa-trie"
}
func validateArithmeticIndexType(indexType string) bool {
// compatible with the index type Asceneding of attu versions prior to 2.3.0
return indexType == DefaultArithmeticIndexType || indexType == "Asceneding"
}
func validateFieldName(fieldName string) error {
fieldName = strings.TrimSpace(fieldName)