fix: add params to ignore config type exception (#41777)

pr: #41776

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>

configs/milvus.yaml

@@ -912,6 +912,7 @@ common:
   enabledOptimizeExpr: true # Indicates whether to enable optimize expr
   enabledJSONKeyStats: false # Indicates sealedsegment whether to enable JSON key stats
   enabledGrowingSegmentJSONKeyStats: false # Indicates growingsegment whether to enable JSON key stats
+  enableConfigParamTypeCheck: true # Indicates whether to enable config param type check
 
 # QuotaConfig, configurations of Milvus quota and limits.
 # By default, we enable:

internal/core/src/common/Common.cpp

@@ -32,6 +32,7 @@ int64_t EXEC_EVAL_EXPR_BATCH_SIZE = DEFAULT_EXEC_EVAL_EXPR_BATCH_SIZE;
 int64_t JSON_KEY_STATS_COMMIT_INTERVAL = DEFAULT_JSON_KEY_STATS_COMMIT_INTERVAL;
 bool OPTIMIZE_EXPR_ENABLED = DEFAULT_OPTIMIZE_EXPR_ENABLED;
 bool GROWING_JSON_KEY_STATS_ENABLED = DEFAULT_GROWING_JSON_KEY_STATS_ENABLED;
+bool CONFIG_PARAM_TYPE_CHECK_ENABLED = DEFAULT_CONFIG_PARAM_TYPE_CHECK_ENABLED;
 
 void
 SetIndexSliceSize(const int64_t size) {
@@ -91,4 +92,10 @@ SetDefaultOptimizeExprEnable(bool val) {
     LOG_INFO("set default optimize expr enabled: {}", OPTIMIZE_EXPR_ENABLED);
 }
 
+void
+SetDefaultConfigParamTypeCheck(bool val) {
+    CONFIG_PARAM_TYPE_CHECK_ENABLED = val;
+    LOG_INFO("set default config param type check enabled: {}",
+             CONFIG_PARAM_TYPE_CHECK_ENABLED);
+}
 }  // namespace milvus

internal/core/src/common/Common.h

@@ -32,6 +32,8 @@ extern int64_t EXEC_EVAL_EXPR_BATCH_SIZE;
 extern int64_t JSON_KEY_STATS_COMMIT_INTERVAL;
 extern bool OPTIMIZE_EXPR_ENABLED;
 extern bool GROWING_JSON_KEY_STATS_ENABLED;
+extern bool CONFIG_PARAM_TYPE_CHECK_ENABLED;
+
 void
 SetIndexSliceSize(const int64_t size);
 
@@ -59,6 +61,9 @@ SetDefaultGrowingJSONKeyStatsEnable(bool val);
 void
 SetDefaultOptimizeExprEnable(bool val);
 
+void
+SetDefaultConfigParamTypeCheck(bool val);
+
 struct BufferView {
     struct Element {
         const char* data_;

internal/core/src/common/Consts.h

@@ -87,3 +87,16 @@ const bool DEFAULT_OPTIMIZE_EXPR_ENABLED = true;
 const bool DEFAULT_GROWING_JSON_KEY_STATS_ENABLED = false;
 const int64_t DEFAULT_JSON_KEY_STATS_COMMIT_INTERVAL = 200;
 const int64_t DEFAULT_CONVERT_OR_TO_IN_NUMERIC_LIMIT = 150;
+const bool DEFAULT_CONFIG_PARAM_TYPE_CHECK_ENABLED = true;
+
+// index config related
+const std::string SEGMENT_INSERT_FILES_KEY = "segment_insert_files";
+const std::string INSERT_FILES_KEY = "insert_files";
+const std::string PARTITION_KEY_ISOLATION_KEY = "partition_key_isolation";
+const std::string STORAGE_VERSION_KEY = "storage_version";
+const std::string DIM_KEY = "dim";
+const std::string DATA_TYPE_KEY = "data_type";
+
+// storage version
+const int64_t STORAGE_V1 = 1;
+const int64_t STORAGE_V2 = 2;

internal/core/src/common/init_c.cpp

@@ -20,7 +20,7 @@
 #include "common/Tracer.h"
 
 std::once_flag flag1, flag2, flag3, flag4, flag5, flag6, flag7, flag8, flag9,
-    flag10;
+    flag10, flag11;
 std::once_flag traceFlag;
 
 void
@@ -95,6 +95,14 @@ InitDefaultOptimizeExprEnable(bool val) {
         val);
 }
 
+void
+InitDefaultConfigParamTypeCheck(bool val) {
+    std::call_once(
+        flag11,
+        [](bool val) { milvus::SetDefaultConfigParamTypeCheck(val); },
+        val);
+}
+
 void
 InitTrace(CTraceConfig* config) {
     auto traceConfig = milvus::tracer::TraceConfig{config->exporter,

internal/core/src/common/init_c.h

@@ -57,6 +57,9 @@ InitDefaultGrowingJSONKeyStatsEnable(bool val);
 void
 InitDefaultOptimizeExprEnable(bool val);
 
+void
+InitDefaultConfigParamTypeCheck(bool val);
+
 #ifdef __cplusplus
 };
 #endif

internal/core/src/index/Utils.h

@@ -28,6 +28,7 @@
 #include <string>
 #include <boost/algorithm/string.hpp>
 
+#include "common/Common.h"
 #include "common/Types.h"
 #include "common/FieldData.h"
 #include "common/QueryInfo.h"
@@ -35,6 +36,7 @@
 #include "index/IndexInfo.h"
 #include "storage/Types.h"
 #include "storage/DataCodec.h"
+#include "log/Log.h"
 
 namespace milvus::index {
 
@@ -83,26 +85,39 @@ void inline CheckParameter(Config& conf,
 template <typename T>
 inline std::optional<T>
 GetValueFromConfig(const Config& cfg, const std::string& key) {
-    if (cfg.contains(key)) {
+    if (!cfg.contains(key)) {
-        if (cfg.at(key).is_null()) {
+        return std::nullopt;
+    }
+
+    const auto& value = cfg.at(key);
+    if (value.is_null()) {
+        return std::nullopt;
+    }
+
+    try {
+        if constexpr (std::is_same_v<T, bool>) {
+            if (value.is_boolean()) {
+                return value.get<bool>();
+            }
+            // compatibility for boolean string
+            return boost::algorithm::to_lower_copy(value.get<std::string>()) ==
+                   "true";
+        }
+        return value.get<T>();
+    } catch (const nlohmann::json::type_error& e) {
+        if (!CONFIG_PARAM_TYPE_CHECK_ENABLED) {
+            LOG_WARN("config type mismatch for key {}: {}", key, e.what());
             return std::nullopt;
         }
-        try {
+        PanicInfo(ErrorCode::UnexpectedError,
-            // compatibility for boolean string
+                  "config type error for key {}: {}",
-            if constexpr (std::is_same_v<T, bool>) {
+                  key,
-                if (cfg.at(key).is_boolean()) {
+                  e.what());
-                    return cfg.at(key).get<bool>();
+    } catch (const std::exception& e) {
-                }
+        PanicInfo(ErrorCode::UnexpectedError,
-                return boost::algorithm::to_lower_copy(
+                  "Unexpected error for key {}: {}",
-                           cfg.at(key).get<std::string>()) == "true";
+                  key,
-            }
+                  e.what());
-            return cfg.at(key).get<T>();
-        } catch (std::exception& e) {
-            PanicInfo(ErrorCode::UnexpectedError,
-                      "get value from config for key {} failed, error: {}",
-                      key,
-                      e.what());
-        }
     }
     return std::nullopt;
 }

internal/core/unittest/test_c_api.cpp

@@ -5302,7 +5302,7 @@ TEST(CApiTest, TestGetValueFromConfig) {
         GetValueFromConfig<std::string>(cfg, "d");
     } catch (const std::exception& e) {
         std::cout << e.what() << std::endl;
-        ASSERT_EQ(std::string(e.what()).find("get value from config for key") !=
+        ASSERT_EQ(std::string(e.what()).find("config type error for key") !=
                       std::string::npos,
                   true);
     }
@@ -5310,3 +5310,26 @@ TEST(CApiTest, TestGetValueFromConfig) {
     auto e_value = GetValueFromConfig<std::string>(cfg, "e");
     ASSERT_FALSE(e_value.has_value());
 }
+
+TEST(CApiTest, TestGetValueFromConfig_Without_Type_Check) {
+    nlohmann::json cfg = nlohmann::json::parse(
+        R"({"a" : 100, "b" : true, "c" : "true", "d" : 1.234, "e" : "1.234", "f" : null})");
+    SetDefaultConfigParamTypeCheck(false);
+    auto a_value = GetValueFromConfig<int64_t>(cfg, "a");
+    ASSERT_EQ(a_value.value(), 100);
+    std::cout << "a_value: " << a_value.value() << std::endl;
+
+    auto b_value = GetValueFromConfig<bool>(cfg, "b");
+    ASSERT_TRUE(b_value.value());
+    std::cout << "b_value: " << b_value.value() << std::endl;
+    auto c_value = GetValueFromConfig<bool>(cfg, "c");
+    ASSERT_TRUE(c_value.value());
+    std::cout << "c_value: " << c_value.value() << std::endl;
+    auto d_value = GetValueFromConfig<double>(cfg, "d");
+    ASSERT_NEAR(d_value.value(), 1.234, 0.001);
+    std::cout << "d_value: " << d_value.value() << std::endl;
+    auto e_value = GetValueFromConfig<double>(cfg, "e");
+    ASSERT_FALSE(e_value.has_value());
+    auto f_value = GetValueFromConfig<bool>(cfg, "f");
+    ASSERT_FALSE(f_value.has_value());
+}

internal/querynodev2/server.go

@@ -263,6 +263,9 @@ func (node *QueryNode) InitSegcore() error {
 	cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
 	C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)
 
+	cEnableConfigParamTypeCheck := C.bool(paramtable.Get().CommonCfg.EnableConfigParamTypeCheck.GetAsBool())
+	C.InitDefaultConfigParamTypeCheck(cEnableConfigParamTypeCheck)
+
 	localDataRootPath := filepath.Join(paramtable.Get().LocalStorageCfg.Path.GetValue(), typeutil.QueryNodeRole)
 	initcore.InitLocalChunkManager(localDataRootPath)
 

pkg/util/paramtable/component_param.go

@@ -300,6 +300,8 @@ type commonConfig struct {
 	EnabledOptimizeExpr               ParamItem `refreshable:"true"`
 	EnabledJSONKeyStats               ParamItem `refreshable:"true"`
 	EnabledGrowingSegmentJSONKeyStats ParamItem `refreshable:"true"`
+
+	EnableConfigParamTypeCheck ParamItem `refreshable:"true"`
 }
 
 func (p *commonConfig) init(base *BaseTable) {
@@ -1028,6 +1030,15 @@ This helps Milvus-CDC synchronize incremental data`,
 		Export:       true,
 	}
 	p.EnabledGrowingSegmentJSONKeyStats.Init(base.mgr)
+
+	p.EnableConfigParamTypeCheck = ParamItem{
+		Key:          "common.enableConfigParamTypeCheck",
+		Version:      "2.5.5",
+		DefaultValue: "true",
+		Doc:          "Indicates whether to enable config param type check",
+		Export:       true,
+	}
+	p.EnableConfigParamTypeCheck.Init(base.mgr)
 }
 
 type gpuConfig struct {