enhance: add param for tuning max VARCHAR length and restore limit to 65535 (#38884)

issue: #38882 

Signed-off-by: Patrick Weizhi Xu <weizhi.xu@zilliz.com>
(cherry picked from commit 8e740e004151dd2c11918aad2857c1c8d1bd98f5)

internal/proxy/task.go

@@ -1256,8 +1256,9 @@ func (t *alterCollectionFieldTask) PreExecute(ctx context.Context) error {
 				return merr.WrapErrParameterInvalid("%s should be an integer, but got %T", prop.Key, prop.Value)
 			}
 
-			if value > defaultMaxVarCharLength {
+			defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
-				return merr.WrapErrParameterInvalid("%s exceeds the maximum allowed value 1048576", prop.Value)
+			if int64(value) > defaultMaxVarCharLength {
+				return merr.WrapErrParameterInvalidMsg("%s exceeds the maximum allowed value %s", prop.Value, strconv.FormatInt(defaultMaxVarCharLength, 10))
 			}
 		}
 	}

internal/proxy/util.go

@@ -63,8 +63,6 @@ const (
 	// enableMultipleVectorFields indicates whether to enable multiple vector fields.
 	enableMultipleVectorFields = true
 
-	defaultMaxVarCharLength = 1048576
-
 	defaultMaxArrayCapacity = 4096
 
 	defaultMaxSearchRequest = 1024
@@ -363,8 +361,10 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema)
 		if err != nil {
 			return err
 		}
+
+		defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
 		if maxLengthPerRow > defaultMaxVarCharLength || maxLengthPerRow <= 0 {
-			return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, 1048576]")
+			return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, %d]", defaultMaxVarCharLength)
 		}
 		exist = true
 	}

pkg/util/paramtable/component_param.go

@@ -1304,6 +1304,7 @@ type proxyConfig struct {
 	SkipAutoIDCheck              ParamItem `refreshable:"true"`
 	SkipPartitionKeyCheck        ParamItem `refreshable:"true"`
 	EnablePublicPrivilege        ParamItem `refreshable:"false"`
+	MaxVarCharLength             ParamItem `refreshable:"false"`
 
 	AccessLog AccessLogConfig
 
@@ -1716,6 +1717,14 @@ please adjust in embedded Milvus: false`,
 	}
 	p.EnablePublicPrivilege.Init(base.mgr)
 
+	p.MaxVarCharLength = ParamItem{
+		Key:          "proxy.maxVarCharLength",
+		Version:      "2.4.19",            // hotfix
+		DefaultValue: strconv.Itoa(65535), // 64K
+		Doc:          "maximum number of characters for a varchar field; this value is overridden by the value in a pre-existing schema if applicable",
+	}
+	p.MaxVarCharLength.Init(base.mgr)
+
 	p.GracefulStopTimeout = ParamItem{
 		Key:          "proxy.gracefulStopTimeout",
 		Version:      "2.3.7",

tests/go_client/common/consts.go

@@ -57,7 +57,7 @@ const (
 	DefaultRgName           = "__default_resource_group"
 	DefaultDb               = "default"
 	MaxDim                  = 32768
-	MaxLength               = int64(1048576)
+	MaxLength               = int64(65535)
 	MaxCollectionNameLen    = 255
 	DefaultRgCapacity       = 1000000
 	RetentionDuration       = 40   // common.retentionDuration

tests/go_client/testcases/collection_test.go

@@ -836,7 +836,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
 	for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
 		arrayVarcharField.WithMaxLength(invalidLength)
 		err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
-		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
+		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
 	}
 }
 
@@ -858,7 +858,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) {
 	for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
 		varcharField.WithMaxLength(invalidLength)
 		err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
-		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
+		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
 	}
 }
 

tests/python_client/testcases/test_collection.py

@@ -3875,10 +3875,10 @@ class TestCollectionString(TestcaseBase):
         c_name = cf.gen_unique_str(prefix)
         int_field = cf.gen_int64_field(is_primary=True)
         vec_field = cf.gen_float_vec_field()
-        max_length = 1048576 + 1
+        max_length = 65535 + 1
         string_field = cf.gen_string_field(max_length=max_length)
         schema = cf.gen_collection_schema([int_field, string_field, vec_field])
-        error = {ct.err_code: 1048576, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 1048576]"}
+        error = {ct.err_code: 65535, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 65535]"}
         self.collection_wrap.init_collection(name=c_name, schema=schema,
                                              check_task=CheckTasks.err_res, check_items=error)
 
@@ -4117,7 +4117,7 @@ class TestCollectionARRAY(TestcaseBase):
         self.init_collection_wrap(schema=array_schema, check_task=CheckTasks.err_res,
                                   check_items={ct.err_code: 65535,
                                                ct.err_msg: "the maximum length specified for a VarChar "
-                                                           "should be in (0, 1048576]"})
+                                                           "should be in (0, 65535]"})
 
     @pytest.mark.tags(CaseLabel.L2)
     def test_collection_array_field_all_datatype(self):