admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: support auto index for array (#35095)

Browse Source 
/kind branch-feature
pr: #34450

---------

Signed-off-by: longjiquan <jiquan.long@zilliz.com>
Co-authored-by: Zhagnlu <lu.zhang@zilliz.com>
This commit is contained in:
Jiquan Long 2024-07-30 17:57:50 +08:00 committed by GitHub
parent d51bfcdd7f
commit 86edca8c1b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 33 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/proxy/accesslog/info/grpc_info.go
View File

@ -22,6 +22,7 @@ import (
"path"
"time"
"github.com/golang/protobuf/proto"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -29,7 +30,6 @@ import (
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/internal/proxy/connection"

35
internal/proxy/task_index.go
View File

@ -160,17 +160,32 @@ func (cit *createIndexTask) parseIndexParams() error {
if !isVecIndex {
specifyIndexType, exist := indexParamsMap[common.IndexTypeKey]
if Params.AutoIndexConfig.ScalarAutoIndexEnable.GetAsBool() || specifyIndexType == AutoIndexName || !exist {
if typeutil.IsArithmetic(cit.fieldSchema.DataType) {
indexParamsMap[common.IndexTypeKey] = Params.AutoIndexConfig.ScalarNumericIndexType.GetValue()
} else if typeutil.IsStringType(cit.fieldSchema.DataType) {
indexParamsMap[common.IndexTypeKey] = Params.AutoIndexConfig.ScalarVarcharIndexType.GetValue()
} else if typeutil.IsBoolType(cit.fieldSchema.DataType) {
indexParamsMap[common.IndexTypeKey] = Params.AutoIndexConfig.ScalarBoolIndexType.GetValue()
} else {
return merr.WrapErrParameterInvalid("supported field",
fmt.Sprintf("create auto index on %s field is not supported", cit.fieldSchema.DataType.String()))
autoIndexEnable := Params.AutoIndexConfig.ScalarAutoIndexEnable.GetAsBool()
if autoIndexEnable || !exist || specifyIndexType == AutoIndexName {
getPrimitiveIndexType := func(dataType schemapb.DataType) (string, error) {
if typeutil.IsArithmetic(dataType) {
return Params.AutoIndexConfig.ScalarNumericIndexType.GetValue(), nil
} else if typeutil.IsStringType(dataType) {
return Params.AutoIndexConfig.ScalarVarcharIndexType.GetValue(), nil
} else if typeutil.IsBoolType(dataType) {
return Params.AutoIndexConfig.ScalarBoolIndexType.GetValue(), nil
}
return "", fmt.Errorf("create auto index on type:%s is not supported", dataType.String())
}
indexType, err := func() (string, error) {
dataType := cit.fieldSchema.DataType
if typeutil.IsArrayType(dataType) {
return getPrimitiveIndexType(cit.fieldSchema.ElementType)
}
return getPrimitiveIndexType(dataType)
}()
if err != nil {
return merr.WrapErrParameterInvalid("supported field", err.Error())
}
indexParamsMap[common.IndexTypeKey] = indexType
}
} else {
specifyIndexType, exist := indexParamsMap[common.IndexTypeKey]

2
internal/querycoordv2/balance/score_based_balancer_test.go
View File

@ -565,7 +565,7 @@ func (suite *ScoreBasedBalancerTestSuite) TestDelegatorPreserveMemory() {
suite.Len(segmentPlans, 0)
paramtable.Get().Save(paramtable.Get().QueryCoordCfg.DelegatorMemoryOverloadFactor.Key, "2")
segmentPlans, channelPlans = suite.getCollectionBalancePlans(balancer, c.collectionID)
segmentPlans, _ = suite.getCollectionBalancePlans(balancer, c.collectionID)
suite.Len(segmentPlans, 1)
suite.Equal(segmentPlans[0].To, int64(2))
})

4
pkg/util/paramtable/component_param.go
View File

@ -1595,7 +1595,7 @@ type queryCoordConfig struct {
RowCountMaxSteps ParamItem `refreshable:"true"`
RandomMaxSteps ParamItem `refreshable:"true"`
GrowingRowCountWeight ParamItem `refreshable:"true"`
DelegatorMemoryOverloadFactor ParamItem `refreshable:"true`
DelegatorMemoryOverloadFactor ParamItem `refreshable:"true"`
BalanceCostThreshold ParamItem `refreshable:"true"`
SegmentCheckInterval ParamItem `refreshable:"true"`
@ -1638,7 +1638,7 @@ type queryCoordConfig struct {
CollectionObserverInterval ParamItem `refreshable:"false"`
CheckExecutedFlagInterval ParamItem `refreshable:"false"`
UpdateCollectionLoadStatusInterval ParamItem `refreshable:"false"`
CollectionBalanceSegmentBatchSize ParamItem `refreshable true`
CollectionBalanceSegmentBatchSize ParamItem `refreshable:"true"`
}
func (p *queryCoordConfig) init(base *BaseTable) {

4
pkg/util/typeutil/schema.go
View File

@ -463,6 +463,10 @@ func IsVariableDataType(dataType schemapb.DataType) bool {
return IsStringType(dataType) || IsArrayType(dataType) || IsJSONType(dataType)
}
func IsPrimitiveType(dataType schemapb.DataType) bool {
return IsArithmetic(dataType) || IsStringType(dataType) || IsBoolType(dataType)
}
// PrepareResultFieldData construct this slice fo FieldData for final result reduce
// this shall preallocate the space for field data internal slice prevent slice growing cost.
func PrepareResultFieldData(sample []*schemapb.FieldData, topK int64) []*schemapb.FieldData {