From 6cbb67832f6ca855b6255c31e3b2e3bb0a9b2cfa Mon Sep 17 00:00:00 2001 From: xige-16 Date: Sun, 15 Oct 2023 13:52:06 +0800 Subject: [PATCH] Compatible with scalar index types marisa-trie and Ascending (#27638) Signed-off-by: xige-16 --- internal/proxy/task_index.go | 16 +++-- internal/proxy/task_index_test.go | 112 ++++++++++++++++++++++++++++++ internal/proxy/util.go | 14 +++- 3 files changed, 135 insertions(+), 7 deletions(-) diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index af8eb06897..38b372f90c 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -140,15 +140,21 @@ func (cit *createIndexTask) parseIndexParams() error { if !isVecIndex { specifyIndexType, exist := indexParamsMap[common.IndexTypeKey] if cit.fieldSchema.DataType == schemapb.DataType_VarChar { - if exist && specifyIndexType != DefaultStringIndexType { + if !exist { + indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType + } + + if exist && !validateStringIndexType(specifyIndexType) { return merr.WrapErrParameterInvalid(DefaultStringIndexType, specifyIndexType, "index type not match") } - indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType } else if typeutil.IsArithmetic(cit.fieldSchema.DataType) { - if exist && specifyIndexType != DefaultIndexType { - return merr.WrapErrParameterInvalid(DefaultIndexType, specifyIndexType, "index type not match") + if !exist { + indexParamsMap[common.IndexTypeKey] = DefaultArithmeticIndexType + } + + if exist && !validateArithmeticIndexType(specifyIndexType) { + return merr.WrapErrParameterInvalid(DefaultArithmeticIndexType, specifyIndexType, "index type not match") } - indexParamsMap[common.IndexTypeKey] = DefaultIndexType } else { return merr.WrapErrParameterInvalid("supported field", fmt.Sprintf("create index on %s field", cit.fieldSchema.DataType.String()), diff --git a/internal/proxy/task_index_test.go b/internal/proxy/task_index_test.go index 687ca626bf..38c8e50794 100644 --- a/internal/proxy/task_index_test.go +++ b/internal/proxy/task_index_test.go @@ -498,6 +498,118 @@ func Test_parseIndexParams(t *testing.T) { assert.Error(t, err) }) + t.Run("create index on VarChar field", func(t *testing.T) { + cit := &createIndexTask{ + req: &milvuspb.CreateIndexRequest{ + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: DefaultStringIndexType, + }, + }, + IndexName: "", + }, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + DataType: schemapb.DataType_VarChar, + }, + } + err := cit.parseIndexParams() + assert.NoError(t, err) + }) + + t.Run("create index on Arithmetic field", func(t *testing.T) { + cit := &createIndexTask{ + req: &milvuspb.CreateIndexRequest{ + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: DefaultArithmeticIndexType, + }, + }, + IndexName: "", + }, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + DataType: schemapb.DataType_Int64, + }, + } + err := cit.parseIndexParams() + assert.NoError(t, err) + }) + + // Compatible with the old version <= 2.3.0 + t.Run("create marisa-trie index on VarChar field", func(t *testing.T) { + cit := &createIndexTask{ + req: &milvuspb.CreateIndexRequest{ + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: "marisa-trie", + }, + }, + IndexName: "", + }, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + DataType: schemapb.DataType_VarChar, + }, + } + err := cit.parseIndexParams() + assert.NoError(t, err) + }) + + // Compatible with the old version <= 2.3.0 + t.Run("create Asceneding index on Arithmetic field", func(t *testing.T) { + cit := &createIndexTask{ + req: &milvuspb.CreateIndexRequest{ + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: "Asceneding", + }, + }, + IndexName: "", + }, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + DataType: schemapb.DataType_Int64, + }, + } + err := cit.parseIndexParams() + assert.NoError(t, err) + }) + + t.Run("create unsupported index on Arithmetic field", func(t *testing.T) { + cit := &createIndexTask{ + req: &milvuspb.CreateIndexRequest{ + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: "invalid_type", + }, + }, + IndexName: "", + }, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + DataType: schemapb.DataType_Int64, + }, + } + err := cit.parseIndexParams() + assert.Error(t, err) + }) + t.Run("create index on array field", func(t *testing.T) { cit3 := &createIndexTask{ Condition: nil, diff --git a/internal/proxy/util.go b/internal/proxy/util.go index 21ddf12872..e12131edca 100644 --- a/internal/proxy/util.go +++ b/internal/proxy/util.go @@ -60,8 +60,8 @@ const ( defaultMaxArrayCapacity = 4096 - // DefaultIndexType name of default index type for scalar field - DefaultIndexType = "STL_SORT" + // DefaultArithmeticIndexType name of default index type for scalar field + DefaultArithmeticIndexType = "STL_SORT" // DefaultStringIndexType name of default index type for varChar/string field DefaultStringIndexType = "Trie" @@ -244,6 +244,16 @@ func validatePartitionTag(partitionTag string, strictCheck bool) error { return nil } +func validateStringIndexType(indexType string) bool { + // compatible with the index type marisa-trie of attu versions prior to 2.3.0 + return indexType == DefaultStringIndexType || indexType == "marisa-trie" +} + +func validateArithmeticIndexType(indexType string) bool { + // compatible with the index type Asceneding of attu versions prior to 2.3.0 + return indexType == DefaultArithmeticIndexType || indexType == "Asceneding" +} + func validateFieldName(fieldName string) error { fieldName = strings.TrimSpace(fieldName)