From d791cb0979bd1d9c0f9ba75ff65bfb2be55102de Mon Sep 17 00:00:00 2001 From: congqixia Date: Fri, 9 May 2025 16:46:54 +0800 Subject: [PATCH] enhance: [AddField] Support explicit dynamic output fields (#41717) Related to #39718 After support add field with dynamic fields enabled, the masked dynamic field shall be able to return with `$meta["name"]` --------- Signed-off-by: Congqi Xia --- internal/proxy/task_test.go | 15 +++++++++++---- internal/proxy/util.go | 34 +++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/internal/proxy/task_test.go b/internal/proxy/task_test.go index dff2b465b2..8a73cf36ca 100644 --- a/internal/proxy/task_test.go +++ b/internal/proxy/task_test.go @@ -644,16 +644,23 @@ func TestTranslateOutputFields(t *testing.T) { assert.ElementsMatch(t, []string{"A"}, userDynamicFields) assert.True(t, requestedPK) - // Test invalid dynamic field expressions - _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"A\"]"}, schema, true) - assert.Error(t, err) + outputFields, userOutputFields, userDynamicFields, requestedPK, err = translateOutputFields([]string{"$meta[\"A\"]", idFieldName}, schema, true) + assert.NoError(t, err) + assert.ElementsMatch(t, []string{common.MetaFieldName}, outputFields) + assert.ElementsMatch(t, []string{"$meta[\"A\"]"}, userOutputFields) + assert.ElementsMatch(t, []string{"A"}, userDynamicFields) + assert.True(t, requestedPK) - _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[]"}, schema, true) + // Test invalid dynamic field expressions + _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, `$meta["A"]["B"]`}, schema, true) assert.Error(t, err) _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[\"\"]"}, schema, true) assert.Error(t, err) + _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta[]"}, schema, true) + assert.Error(t, err) + _, _, _, _, err = translateOutputFields([]string{idFieldName, floatVectorFieldName, "$meta["}, schema, true) assert.Error(t, err) diff --git a/internal/proxy/util.go b/internal/proxy/util.go index 0dc5b89bc2..31417b6c73 100644 --- a/internal/proxy/util.go +++ b/internal/proxy/util.go @@ -1522,24 +1522,36 @@ func translateOutputFields(outputFields []string, schema *schemaInfo, removePkFi } else { if schema.EnableDynamicField { if schema.IsFieldLoaded(dynamicField.GetFieldID()) { - schemaH, err := typeutil.CreateSchemaHelper(schema.CollectionSchema) - if err != nil { - return nil, nil, nil, false, err - } - err = planparserv2.ParseIdentifier(schemaH, outputFieldName, func(expr *planpb.Expr) error { - if len(expr.GetColumnExpr().GetInfo().GetNestedPath()) == 1 && - expr.GetColumnExpr().GetInfo().GetNestedPath()[0] == outputFieldName { - return nil + dynamicNestedPath := outputFieldName + err := planparserv2.ParseIdentifier(schema.schemaHelper, outputFieldName, func(expr *planpb.Expr) error { + columnInfo := expr.GetColumnExpr().GetInfo() + // there must be no error here + dynamicField, _ := schema.schemaHelper.GetDynamicField() + // only $meta["xxx"] is allowed for now + if dynamicField.GetFieldID() != columnInfo.GetFieldId() { + return errors.New("not support getting subkeys of json field yet") } - return errors.New("not support getting subkeys of json field yet") + nestedPaths := columnInfo.GetNestedPath() + // $meta["A"]["B"] not allowed for now + if len(nestedPaths) != 1 { + return errors.New("not support getting multiple level of dynamic field for now") + } + // $meta["dyn_field"], output field name could be: + // 1. "dyn_field", outputFieldName == nestedPath + // 2. `$meta["dyn_field"]` explicit form + if nestedPaths[0] != outputFieldName { + // use "dyn_field" as userDynamicFieldsMap when outputField = `$meta["dyn_field"]` + dynamicNestedPath = nestedPaths[0] + } + return nil }) if err != nil { - log.Info("parse output field name failed", zap.String("field name", outputFieldName)) + log.Info("parse output field name failed", zap.String("field name", outputFieldName), zap.Error(err)) return nil, nil, nil, false, fmt.Errorf("parse output field name failed: %s", outputFieldName) } resultFieldNameMap[common.MetaFieldName] = true userOutputFieldsMap[outputFieldName] = true - userDynamicFieldsMap[outputFieldName] = true + userDynamicFieldsMap[dynamicNestedPath] = true } else { // TODO after cold field be able to fetched with chunk cache, this check shall be removed return nil, nil, nil, false, fmt.Errorf("field %s cannot be returned since dynamic field not loaded", outputFieldName)