mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 17:48:29 +08:00
enhance: [2.5] avoid re-query if hybrid search requested only pk as output field (#40906)
pr: https://github.com/milvus-io/milvus/pull/40842 issue: https://github.com/milvus-io/milvus/issues/40833 --------- Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng
GitHub
parent
27ea5d14dc
commit
8504a7b98d
@ -85,11 +85,12 @@ type queryParams struct {
|
||||
}
|
||||
|
||||
// translateToOutputFieldIDs translates output fields name to output fields id.
|
||||
// If no output fields specified, return only pk field
|
||||
func translateToOutputFieldIDs(outputFields []string, schema *schemapb.CollectionSchema) ([]UniqueID, error) {
|
||||
outputFieldIDs := make([]UniqueID, 0, len(outputFields)+1)
|
||||
if len(outputFields) == 0 {
|
||||
for _, field := range schema.Fields {
|
||||
if field.FieldID >= common.StartOfUserFieldID && !typeutil.IsVectorType(field.DataType) {
|
||||
if field.IsPrimaryKey {
|
||||
outputFieldIDs = append(outputFieldIDs, field.FieldID)
|
||||
}
|
||||
}
|
||||
@ -270,7 +271,7 @@ func (t *queryTask) createPlan(ctx context.Context) error {
|
||||
metrics.ProxyParseExpressionLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "query", metrics.SuccessLabel).Observe(float64(time.Since(start).Milliseconds()))
|
||||
}
|
||||
|
||||
t.request.OutputFields, t.userOutputFields, t.userDynamicFields, err = translateOutputFields(t.request.OutputFields, t.schema, true)
|
||||
t.request.OutputFields, t.userOutputFields, t.userDynamicFields, _, err = translateOutputFields(t.request.OutputFields, t.schema, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -404,7 +404,7 @@ func Test_translateToOutputFieldIDs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
expectedIDs: []int64{100, 101},
|
||||
expectedIDs: []int64{100},
|
||||
},
|
||||
{
|
||||
name: "nil output fields",
|
||||
@ -427,7 +427,7 @@ func Test_translateToOutputFieldIDs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
expectedIDs: []int64{100, 101},
|
||||
expectedIDs: []int64{100},
|
||||
},
|
||||
{
|
||||
name: "full list",
|
||||
|
||||
@ -88,6 +88,9 @@ type searchTask struct {
|
||||
groupScorer func(group *Group) error
|
||||
|
||||
isIterator bool
|
||||
// we always remove pk field from output fields, as search result already contains pk field.
|
||||
// if the user explicitly set pk field in output fields, we add it back to the result.
|
||||
userRequestedPkFieldExplicitly bool
|
||||
}
|
||||
|
||||
func (t *searchTask) CanSkipAllocTimestamp() bool {
|
||||
@ -163,7 +166,7 @@ func (t *searchTask) PreExecute(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
t.request.OutputFields, t.userOutputFields, t.userDynamicFields, err = translateOutputFields(t.request.OutputFields, t.schema, false)
|
||||
t.request.OutputFields, t.userOutputFields, t.userDynamicFields, t.userRequestedPkFieldExplicitly, err = translateOutputFields(t.request.OutputFields, t.schema, true)
|
||||
if err != nil {
|
||||
log.Warn("translate output fields failed", zap.Error(err))
|
||||
return err
|
||||
@ -786,6 +789,33 @@ func (t *searchTask) PostExecute(ctx context.Context) error {
|
||||
t.result.Results.OutputFields = t.userOutputFields
|
||||
t.result.CollectionName = t.request.GetCollectionName()
|
||||
t.result.Results.PrimaryFieldName = primaryFieldSchema.GetName()
|
||||
if t.userRequestedPkFieldExplicitly {
|
||||
t.result.Results.OutputFields = append(t.result.Results.OutputFields, primaryFieldSchema.GetName())
|
||||
var scalars *schemapb.ScalarField
|
||||
if primaryFieldSchema.GetDataType() == schemapb.DataType_Int64 {
|
||||
scalars = &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{
|
||||
LongData: t.result.Results.Ids.GetIntId(),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
scalars = &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_StringData{
|
||||
StringData: t.result.Results.Ids.GetStrId(),
|
||||
},
|
||||
}
|
||||
}
|
||||
pkFieldData := &schemapb.FieldData{
|
||||
FieldName: primaryFieldSchema.GetName(),
|
||||
FieldId: primaryFieldSchema.GetFieldID(),
|
||||
Type: primaryFieldSchema.GetDataType(),
|
||||
IsDynamic: false,
|
||||
Field: &schemapb.FieldData_Scalars{
|
||||
Scalars: scalars,
|
||||
},
|
||||
}
|
||||
t.result.Results.FieldsData = append(t.result.Results.FieldsData, pkFieldData)
|
||||
}
|
||||
if t.isIterator && len(t.queryInfos) == 1 && t.queryInfos[0] != nil {
|
||||
if iterInfo := t.queryInfos[0].GetSearchIteratorV2Info(); iterInfo != nil {
|
||||
t.result.Results.SearchIteratorV2Results = &schemapb.SearchIteratorV2Results{
|
||||
|
||||
@ -472,6 +472,7 @@ func TestTranslateOutputFields(t *testing.T) {
|
||||
var outputFields []string
|
||||
var userOutputFields []string
|
||||
var userDynamicFields []string
|
||||
var requestedPK bool
|
||||
var err error
|
||||
|
||||
collSchema := &schemapb.CollectionSchema{
|
||||
@ -498,107 +499,124 @@ func TestTranslateOutputFields(t *testing.T) {
|
||||
}
|
||||
schema := newSchemaInfo(collSchema)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test empty output fields
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{}, outputFields)
|
||||
assert.ElementsMatch(t, []string{}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.False(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test single field
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{idFieldName}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, tsFieldName}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test multiple fields
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{idFieldName, tsFieldName}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, tsFieldName, floatVectorFieldName}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test with vector field
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{idFieldName, tsFieldName, floatVectorFieldName}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
// sparse_float_vector is a BM25 function output field, so it should not be included in the output fields
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*"}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test without id field
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{tsFieldName, floatVectorFieldName}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.False(t, requestedPK)
|
||||
|
||||
// Test wildcard - should not include function output fields
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"*"}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{" * "}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
// Test wildcard with spaces
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{" * "}, schema, false)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, false)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
|
||||
// sparse_float_vector is a BM25 function output field, so it should not be included in the output fields
|
||||
_, _, _, err = translateOutputFields([]string{"*", sparseFloatVectorFieldName}, schema, false)
|
||||
// Test function output field - should error
|
||||
_, _, _, _, err = translateOutputFields([]string{"*", sparseFloatVectorFieldName}, schema, false)
|
||||
assert.Error(t, err)
|
||||
_, _, _, err = translateOutputFields([]string{sparseFloatVectorFieldName}, schema, false)
|
||||
_, _, _, _, err = translateOutputFields([]string{sparseFloatVectorFieldName}, schema, false)
|
||||
assert.Error(t, err)
|
||||
_, _, _, err = translateOutputFields([]string{sparseFloatVectorFieldName}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{sparseFloatVectorFieldName}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
//=========================================================================
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, userOutputFields)
|
||||
// Test with removePkField=true
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{}, schema, true)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{}, outputFields)
|
||||
assert.ElementsMatch(t, []string{}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.False(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName}, userOutputFields)
|
||||
// if removePkField is true, pk field should be removed from output fields
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"*"}, schema, true)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, tsFieldName}, schema, true)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{idFieldName, tsFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, tsFieldName, floatVectorFieldName}, schema, true)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{idFieldName, tsFieldName, floatVectorFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*"}, schema, true)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"*"}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, true)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"*", tsFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, true)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName, bfloat16VectorFieldName}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
_, _, _, err = translateOutputFields([]string{"A"}, schema, true)
|
||||
// Test non-existent field, dynamic field not enabled
|
||||
_, _, _, _, err = translateOutputFields([]string{"A"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
t.Run("enable dynamic schema", func(t *testing.T) {
|
||||
@ -617,30 +635,33 @@ func TestTranslateOutputFields(t *testing.T) {
|
||||
}
|
||||
schema := newSchemaInfo(collSchema)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{"A", idFieldName}, schema, true)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.ElementsMatch(t, []string{common.MetaFieldName, idFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{"A", idFieldName}, userOutputFields)
|
||||
outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"A", idFieldName}, schema, true)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{common.MetaFieldName}, outputFields)
|
||||
assert.ElementsMatch(t, []string{"A"}, userOutputFields)
|
||||
assert.ElementsMatch(t, []string{"A"}, userDynamicFields)
|
||||
assert.True(t, requestedPK)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"A\"]"}, schema, true)
|
||||
// Test invalid dynamic field expressions
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"A\"]"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[]"}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[]"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"\"]"}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"\"]"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta["}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta["}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "[]"}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "[]"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "A > 1"}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "A > 1"}, schema, true)
|
||||
assert.Error(t, err)
|
||||
|
||||
outputFields, userOutputFields, userDynamicFields, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, ""}, schema, true)
|
||||
_, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, ""}, schema, true)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
@ -1308,7 +1308,11 @@ func computeRecall(results *schemapb.SearchResultData, gts *schemapb.SearchResul
|
||||
// output_fields=["*"] ==> [A,B,C,D]
|
||||
// output_fields=["*",A] ==> [A,B,C,D]
|
||||
// output_fields=["*",C] ==> [A,B,C,D]
|
||||
func translateOutputFields(outputFields []string, schema *schemaInfo, addPrimary bool) ([]string, []string, []string, error) {
|
||||
//
|
||||
// 4th return value is true if user requested pk field explicitly or using wildcard.
|
||||
// if removePkField is true, pk field will not be include in the first(resultFieldNames)/second(userOutputFields)
|
||||
// return value.
|
||||
func translateOutputFields(outputFields []string, schema *schemaInfo, removePkField bool) ([]string, []string, []string, bool, error) {
|
||||
var primaryFieldName string
|
||||
var dynamicField *schemapb.FieldSchema
|
||||
allFieldNameMap := make(map[string]*schemapb.FieldSchema)
|
||||
@ -1329,9 +1333,15 @@ func translateOutputFields(outputFields []string, schema *schemaInfo, addPrimary
|
||||
allFieldNameMap[field.Name] = field
|
||||
}
|
||||
|
||||
userRequestedPkFieldExplicitly := false
|
||||
|
||||
for _, outputFieldName := range outputFields {
|
||||
outputFieldName = strings.TrimSpace(outputFieldName)
|
||||
if outputFieldName == primaryFieldName {
|
||||
userRequestedPkFieldExplicitly = true
|
||||
}
|
||||
if outputFieldName == "*" {
|
||||
userRequestedPkFieldExplicitly = true
|
||||
for fieldName, field := range allFieldNameMap {
|
||||
// skip Cold field and fields that can't be output
|
||||
if schema.IsFieldLoaded(field.GetFieldID()) && schema.CanRetrieveRawFieldData(field) {
|
||||
@ -1343,20 +1353,20 @@ func translateOutputFields(outputFields []string, schema *schemaInfo, addPrimary
|
||||
} else {
|
||||
if field, ok := allFieldNameMap[outputFieldName]; ok {
|
||||
if !schema.CanRetrieveRawFieldData(field) {
|
||||
return nil, nil, nil, fmt.Errorf("not allowed to retrieve raw data of field %s", outputFieldName)
|
||||
return nil, nil, nil, false, fmt.Errorf("not allowed to retrieve raw data of field %s", outputFieldName)
|
||||
}
|
||||
if schema.IsFieldLoaded(field.GetFieldID()) {
|
||||
resultFieldNameMap[outputFieldName] = true
|
||||
userOutputFieldsMap[outputFieldName] = true
|
||||
} else {
|
||||
return nil, nil, nil, fmt.Errorf("field %s is not loaded", outputFieldName)
|
||||
return nil, nil, nil, false, fmt.Errorf("field %s is not loaded", outputFieldName)
|
||||
}
|
||||
} else {
|
||||
if schema.EnableDynamicField {
|
||||
if schema.IsFieldLoaded(dynamicField.GetFieldID()) {
|
||||
schemaH, err := typeutil.CreateSchemaHelper(schema.CollectionSchema)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, nil, false, err
|
||||
}
|
||||
err = planparserv2.ParseIdentifier(schemaH, outputFieldName, func(expr *planpb.Expr) error {
|
||||
if len(expr.GetColumnExpr().GetInfo().GetNestedPath()) == 1 &&
|
||||
@ -1367,25 +1377,25 @@ func translateOutputFields(outputFields []string, schema *schemaInfo, addPrimary
|
||||
})
|
||||
if err != nil {
|
||||
log.Info("parse output field name failed", zap.String("field name", outputFieldName))
|
||||
return nil, nil, nil, fmt.Errorf("parse output field name failed: %s", outputFieldName)
|
||||
return nil, nil, nil, false, fmt.Errorf("parse output field name failed: %s", outputFieldName)
|
||||
}
|
||||
resultFieldNameMap[common.MetaFieldName] = true
|
||||
userOutputFieldsMap[outputFieldName] = true
|
||||
userDynamicFieldsMap[outputFieldName] = true
|
||||
} else {
|
||||
// TODO after cold field be able to fetched with chunk cache, this check shall be removed
|
||||
return nil, nil, nil, fmt.Errorf("field %s cannot be returned since dynamic field not loaded", outputFieldName)
|
||||
return nil, nil, nil, false, fmt.Errorf("field %s cannot be returned since dynamic field not loaded", outputFieldName)
|
||||
}
|
||||
} else {
|
||||
return nil, nil, nil, fmt.Errorf("field %s not exist", outputFieldName)
|
||||
return nil, nil, nil, false, fmt.Errorf("field %s not exist", outputFieldName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if addPrimary {
|
||||
resultFieldNameMap[primaryFieldName] = true
|
||||
userOutputFieldsMap[primaryFieldName] = true
|
||||
if removePkField {
|
||||
delete(resultFieldNameMap, primaryFieldName)
|
||||
delete(userOutputFieldsMap, primaryFieldName)
|
||||
}
|
||||
|
||||
for fieldName := range resultFieldNameMap {
|
||||
@ -1400,7 +1410,7 @@ func translateOutputFields(outputFields []string, schema *schemaInfo, addPrimary
|
||||
}
|
||||
}
|
||||
|
||||
return resultFieldNames, userOutputFields, userDynamicFields, nil
|
||||
return resultFieldNames, userOutputFields, userDynamicFields, userRequestedPkFieldExplicitly, nil
|
||||
}
|
||||
|
||||
func validateIndexName(indexName string) error {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user