Check the length of JSON (#24035)

Signed-off-by: yah01 <yang.cen@zilliz.com>

internal/proxy/validate_util.go

@@ -6,6 +6,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
 	"github.com/milvus-io/milvus/pkg/util/funcutil"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
 )
 
@@ -63,8 +64,11 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col
 			if err := v.checkVarCharFieldData(field, fieldSchema); err != nil {
 				return err
 			}
+		case schemapb.DataType_JSON:
+			if err := v.checkJSONFieldData(field, fieldSchema); err != nil {
+				return err
+			}
 		default:
-
 		}
 	}
 
@@ -172,7 +176,21 @@ func (v *validateUtil) checkVarCharFieldData(field *schemapb.FieldData, fieldSch
 	return nil
 }
 
-func verifyLengthPerRow(strArr []string, maxLength int64) error {
+func (v *validateUtil) checkJSONFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
+	jsonArray := field.GetScalars().GetJsonData().GetData()
+	if jsonArray == nil {
+		msg := fmt.Sprintf("varchar field '%v' is illegal, array type mismatch", field.GetFieldName())
+		return merr.WrapErrParameterInvalid("need string array", "got nil", msg)
+	}
+
+	if v.checkMaxLen {
+		return verifyLengthPerRow(jsonArray, paramtable.Get().CommonCfg.JSONMaxLength.GetAsInt64())
+	}
+
+	return nil
+}
+
+func verifyLengthPerRow[E interface{ ~string | ~[]byte }](strArr []E, maxLength int64) error {
 	for i, s := range strArr {
 		if int64(len(s)) > maxLength {
 			msg := fmt.Sprintf("the length (%d) of %dth string exceeds max length (%d)", len(s), i, maxLength)

internal/proxy/validate_util_test.go

@@ -8,13 +8,14 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
 )
 
 func Test_verifyLengthPerRow(t *testing.T) {
 	maxLength := 16
 
-	assert.NoError(t, verifyLengthPerRow(nil, int64(maxLength)))
+	assert.NoError(t, verifyLengthPerRow[string](nil, int64(maxLength)))
 
 	assert.NoError(t, verifyLengthPerRow([]string{"111111", "22222"}, int64(maxLength)))
 
@@ -596,6 +597,8 @@ func Test_validateUtil_checkAligned(t *testing.T) {
 }
 
 func Test_validateUtil_Validate(t *testing.T) {
+	paramtable.Init()
+
 	t.Run("nil schema", func(t *testing.T) {
 		data := []*schemapb.FieldData{
 			{
@@ -816,9 +819,36 @@ func Test_validateUtil_Validate(t *testing.T) {
 		}
 
 		v := newValidateUtil(withNANCheck(), withMaxLenCheck())
-
 		err := v.Validate(data, schema, 2)
+		assert.Error(t, err)
 
+		// Validate JSON length
+		longBytes := make([]byte, paramtable.Get().CommonCfg.JSONMaxLength.GetAsInt()+1)
+		data = []*schemapb.FieldData{
+			{
+				FieldName: "json",
+				Type:      schemapb.DataType_JSON,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_JsonData{
+							JsonData: &schemapb.JSONArray{
+								Data: [][]byte{longBytes, longBytes},
+							},
+						},
+					},
+				},
+			},
+		}
+		schema = &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "json",
+					FieldID:  104,
+					DataType: schemapb.DataType_JSON,
+				},
+			},
+		}
+		err = v.Validate(data, schema, 2)
 		assert.Error(t, err)
 	})
 
@@ -831,7 +861,7 @@ func Test_validateUtil_Validate(t *testing.T) {
 					Vectors: &schemapb.VectorField{
 						Data: &schemapb.VectorField_FloatVector{
 							FloatVector: &schemapb.FloatArray{
-								Data: generateFloatVectors(10, 8),
+								Data: generateFloatVectors(2, 8),
 							},
 						},
 					},
@@ -843,7 +873,7 @@ func Test_validateUtil_Validate(t *testing.T) {
 				Field: &schemapb.FieldData_Vectors{
 					Vectors: &schemapb.VectorField{
 						Data: &schemapb.VectorField_BinaryVector{
-							BinaryVector: generateBinaryVectors(10, 8),
+							BinaryVector: generateBinaryVectors(2, 8),
 						},
 					},
 				},
@@ -855,7 +885,20 @@ func Test_validateUtil_Validate(t *testing.T) {
 					Scalars: &schemapb.ScalarField{
 						Data: &schemapb.ScalarField_StringData{
 							StringData: &schemapb.StringArray{
-								Data: generateVarCharArray(10, 8),
+								Data: generateVarCharArray(2, 8),
+							},
+						},
+					},
+				},
+			},
+			{
+				FieldName: "test4",
+				Type:      schemapb.DataType_JSON,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_JsonData{
+							JsonData: &schemapb.JSONArray{
+								Data: [][]byte{[]byte("{}"), []byte("{}")},
 							},
 						},
 					},
@@ -898,12 +941,17 @@ func Test_validateUtil_Validate(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name:     "test4",
+					FieldID:  104,
+					DataType: schemapb.DataType_JSON,
+				},
 			},
 		}
 
 		v := newValidateUtil(withNANCheck(), withMaxLenCheck())
 
-		err := v.Validate(data, schema, 10)
+		err := v.Validate(data, schema, 2)
 
 		assert.NoError(t, err)
 	})

pkg/util/paramtable/component_param.go

@@ -214,6 +214,8 @@ type commonConfig struct {
 	PreCreatedTopicEnabled ParamItem `refreshable:"true"`
 	TopicNames             ParamItem `refreshable:"true"`
 	TimeTicker             ParamItem `refreshable:"true"`
+
+	JSONMaxLength ParamItem `refreshable:"false"`
 }
 
 func (p *commonConfig) init(base *BaseTable) {
@@ -634,6 +636,13 @@ like the old password verification when updating the credential`,
 		Version: "2.3.0",
 	}
 	p.TimeTicker.Init(base.mgr)
+
+	p.JSONMaxLength = ParamItem{
+		Key:          "common.JSONMaxLength",
+		Version:      "2.2.9",
+		DefaultValue: fmt.Sprint(64 << 10),
+	}
+	p.JSONMaxLength.Init(base.mgr)
 }
 
 type traceConfig struct {