From 948d04cdd8b9d79e8e6f42d4054906eed2ca3ec4 Mon Sep 17 00:00:00 2001 From: smellthemoon <64083300+smellthemoon@users.noreply.github.com> Date: Mon, 3 Jul 2023 15:26:26 +0800 Subject: [PATCH] Check field type when create index (#25248) Signed-off-by: lixinguo Co-authored-by: lixinguo --- internal/proxy/task_index.go | 28 +++--- internal/proxy/task_index_test.go | 93 ++++++++++++++++++++ tests/python_client/testcases/test_search.py | 1 + 3 files changed, 112 insertions(+), 10 deletions(-) diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index 7735fdbb99..123bdb9cd9 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/indexparamcheck" "github.com/milvus-io/milvus/pkg/util/indexparams" + "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -122,16 +123,6 @@ func (cit *createIndexTask) parseIndexParams() error { isVecIndex := typeutil.IsVectorType(cit.fieldSchema.DataType) indexParamsMap := make(map[string]string) - if !isVecIndex { - if cit.fieldSchema.DataType == schemapb.DataType_VarChar { - indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType - } else { - indexParamsMap[common.IndexTypeKey] = DefaultIndexType - } - } - if cit.fieldSchema.DataType == schemapb.DataType_JSON { - return fmt.Errorf("create index on json field is not supported") - } for _, kv := range cit.req.GetExtraParams() { if kv.Key == common.IndexParamsKey { @@ -146,6 +137,23 @@ func (cit *createIndexTask) parseIndexParams() error { indexParamsMap[kv.Key] = kv.Value } } + if !isVecIndex { + specifyIndexType, exist := indexParamsMap[common.IndexTypeKey] + if cit.fieldSchema.DataType == schemapb.DataType_VarChar { + if exist && specifyIndexType != DefaultStringIndexType { + return merr.WrapErrParameterInvalid(DefaultStringIndexType, specifyIndexType, "index type not match") + } + indexParamsMap[common.IndexTypeKey] = DefaultStringIndexType + } else { + if cit.fieldSchema.DataType == schemapb.DataType_JSON { + return merr.WrapErrParameterInvalid("not json field", "create index on json field", "create index on json field is not supported") + } + if exist && specifyIndexType != DefaultIndexType { + return merr.WrapErrParameterInvalid(DefaultStringIndexType, specifyIndexType, "index type not match") + } + indexParamsMap[common.IndexTypeKey] = DefaultIndexType + } + } if isVecIndex { specifyIndexType, exist := indexParamsMap[common.IndexTypeKey] diff --git a/internal/proxy/task_index_test.go b/internal/proxy/task_index_test.go index 9078499549..8658600903 100644 --- a/internal/proxy/task_index_test.go +++ b/internal/proxy/task_index_test.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/config" "github.com/milvus-io/milvus/pkg/util/funcutil" + "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -502,6 +503,98 @@ func Test_parseIndexParams(t *testing.T) { err := cit3.parseIndexParams() assert.Error(t, err) }) + + t.Run("pass vector index type on scalar field", func(t *testing.T) { + cit4 := &createIndexTask{ + Condition: nil, + req: &milvuspb.CreateIndexRequest{ + Base: nil, + DbName: "", + CollectionName: "", + FieldName: "", + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: "HNSW", + }, + { + Key: MetricTypeKey, + Value: "IP", + }, + { + Key: common.IndexParamsKey, + Value: "{\"M\": 48, \"efConstruction\": 64}", + }, + { + Key: DimKey, + Value: "128", + }, + }, + IndexName: "", + }, + ctx: nil, + rootCoord: nil, + result: nil, + isAutoIndex: false, + newIndexParams: nil, + newTypeParams: nil, + collectionID: 0, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + Description: "field no.1", + DataType: schemapb.DataType_VarChar, + }, + } + err := cit4.parseIndexParams() + assert.ErrorIs(t, err, merr.ErrParameterInvalid) + + cit5 := &createIndexTask{ + Condition: nil, + req: &milvuspb.CreateIndexRequest{ + Base: nil, + DbName: "", + CollectionName: "", + FieldName: "", + ExtraParams: []*commonpb.KeyValuePair{ + { + Key: common.IndexTypeKey, + Value: "HNSW", + }, + { + Key: MetricTypeKey, + Value: "IP", + }, + { + Key: common.IndexParamsKey, + Value: "{\"M\": 48, \"efConstruction\": 64}", + }, + { + Key: DimKey, + Value: "128", + }, + }, + IndexName: "", + }, + ctx: nil, + rootCoord: nil, + result: nil, + isAutoIndex: false, + newIndexParams: nil, + newTypeParams: nil, + collectionID: 0, + fieldSchema: &schemapb.FieldSchema{ + FieldID: 101, + Name: "FieldID", + IsPrimaryKey: false, + Description: "field no.1", + DataType: schemapb.DataType_Int64, + }, + } + err = cit5.parseIndexParams() + assert.ErrorIs(t, err, merr.ErrParameterInvalid) + }) } func Test_wrapUserIndexParams(t *testing.T) { diff --git a/tests/python_client/testcases/test_search.py b/tests/python_client/testcases/test_search.py index 80d9b6d403..1542ec82bb 100644 --- a/tests/python_client/testcases/test_search.py +++ b/tests/python_client/testcases/test_search.py @@ -3885,6 +3885,7 @@ class TestCollectionSearch(TestcaseBase): @pytest.mark.tags(CaseLabel.L1) @pytest.mark.parametrize("name", ["_co11ection", "co11_ection"]) @pytest.mark.parametrize("index_name", ["_1ndeX", "In_0"]) + @pytest.mark.skip(reason="create vector index type on scalar field, issue#25170 @Nico") def test_search_collection_naming_rules(self, nq, dim, name, index_name, _async): """ target: test search collection naming rules