admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add params to ignore config type exception (#41776)

Browse Source 
#41707

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2025-05-13 13:48:56 +08:00 committed by GitHub
parent 61b6ca5b73
commit f094d026f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 98 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configs/milvus.yaml
View File

@ -936,6 +936,7 @@ common:
enabledOptimizeExpr: true # Indicates whether to enable optimize expr
enabledJSONKeyStats: false # Indicates sealedsegment whether to enable JSON key stats
enabledGrowingSegmentJSONKeyStats: false # Indicates growingsegment whether to enable JSON key stats
enableConfigParamTypeCheck: true # Indicates whether to enable config param type check
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:

7
internal/core/src/common/Common.cpp
View File

@ -32,6 +32,7 @@ bool OPTIMIZE_EXPR_ENABLED = DEFAULT_OPTIMIZE_EXPR_ENABLED;
int64_t JSON_KEY_STATS_COMMIT_INTERVAL = DEFAULT_JSON_KEY_STATS_COMMIT_INTERVAL;
bool GROWING_JSON_KEY_STATS_ENABLED = DEFAULT_GROWING_JSON_KEY_STATS_ENABLED;
bool CONFIG_PARAM_TYPE_CHECK_ENABLED = DEFAULT_CONFIG_PARAM_TYPE_CHECK_ENABLED;
void
SetIndexSliceSize(const int64_t size) {
@ -91,4 +92,10 @@ SetDefaultGrowingJSONKeyStatsEnable(bool val) {
GROWING_JSON_KEY_STATS_ENABLED);
}
void
SetDefaultConfigParamTypeCheck(bool val) {
CONFIG_PARAM_TYPE_CHECK_ENABLED = val;
LOG_INFO("set default config param type check enabled: {}",
CONFIG_PARAM_TYPE_CHECK_ENABLED);
}
} // namespace milvus

5
internal/core/src/common/Common.h
View File

@ -32,6 +32,8 @@ extern int64_t EXEC_EVAL_EXPR_BATCH_SIZE;
extern int64_t JSON_KEY_STATS_COMMIT_INTERVAL;
extern bool OPTIMIZE_EXPR_ENABLED;
extern bool GROWING_JSON_KEY_STATS_ENABLED;
extern bool CONFIG_PARAM_TYPE_CHECK_ENABLED;
void
SetIndexSliceSize(const int64_t size);
@ -59,6 +61,9 @@ SetDefaultJSONKeyStatsCommitInterval(int64_t val);
void
SetDefaultGrowingJSONKeyStatsEnable(bool val);
void
SetDefaultConfigParamTypeCheck(bool val);
struct BufferView {
struct Element {
const char* data_;

1
internal/core/src/common/Consts.h
View File

@ -86,6 +86,7 @@ const int64_t DEFAULT_CONVERT_OR_TO_IN_NUMERIC_LIMIT = 150;
const int64_t DEFAULT_JSON_INDEX_MEMORY_BUDGET = 16777216; // bytes, 16MB
const bool DEFAULT_GROWING_JSON_KEY_STATS_ENABLED = false;
const int64_t DEFAULT_JSON_KEY_STATS_COMMIT_INTERVAL = 200;
const bool DEFAULT_CONFIG_PARAM_TYPE_CHECK_ENABLED = true;
// index config related
const std::string SEGMENT_INSERT_FILES_KEY = "segment_insert_files";

11
internal/core/src/common/init_c.cpp
View File

@ -25,7 +25,8 @@
#include "common/Tracer.h"
#include "log/Log.h"
std::once_flag flag1, flag2, flag3, flag4, flag5, flag6, flag7, flag8, flag9;
std::once_flag flag1, flag2, flag3, flag4, flag5, flag6, flag7, flag8, flag9,
flag10;
std::once_flag traceFlag;
void
@ -102,6 +103,14 @@ InitDefaultGrowingJSONKeyStatsEnable(bool val) {
val);
}
void
InitDefaultConfigParamTypeCheck(bool val) {
std::call_once(
flag10,
[](bool val) { milvus::SetDefaultConfigParamTypeCheck(val); },
val);
}
void
InitTrace(CTraceConfig* config) {
auto traceConfig = milvus::tracer::TraceConfig{config->exporter,

3
internal/core/src/common/init_c.h
View File

@ -57,6 +57,9 @@ InitDefaultJSONKeyStatsCommitInterval(int64_t val);
void
InitDefaultGrowingJSONKeyStatsEnable(bool val);
void
InitDefaultConfigParamTypeCheck(bool val);
#ifdef __cplusplus
};
#endif

51
internal/core/src/index/Utils.h
View File

@ -28,6 +28,7 @@
#include <string>
#include <boost/algorithm/string.hpp>
#include "common/Common.h"
#include "common/Types.h"
#include "common/FieldData.h"
#include "common/QueryInfo.h"
@ -35,6 +36,7 @@
#include "index/IndexInfo.h"
#include "storage/Types.h"
#include "storage/DataCodec.h"
#include "log/Log.h"
namespace milvus::index {
@ -83,26 +85,39 @@ void inline CheckParameter(Config& conf,
template <typename T>
inline std::optional<T>
GetValueFromConfig(const Config& cfg, const std::string& key) {
if (cfg.contains(key)) {
if (cfg.at(key).is_null()) {
if (!cfg.contains(key)) {
return std::nullopt;
}
const auto& value = cfg.at(key);
if (value.is_null()) {
return std::nullopt;
}
try {
if constexpr (std::is_same_v<T, bool>) {
if (value.is_boolean()) {
return value.get<bool>();
}
// compatibility for boolean string
return boost::algorithm::to_lower_copy(value.get<std::string>()) ==
"true";
}
return value.get<T>();
} catch (const nlohmann::json::type_error& e) {
if (!CONFIG_PARAM_TYPE_CHECK_ENABLED) {
LOG_WARN("config type mismatch for key {}: {}", key, e.what());
return std::nullopt;
}
try {
// compatibility for boolean string
if constexpr (std::is_same_v<T, bool>) {
if (cfg.at(key).is_boolean()) {
return cfg.at(key).get<bool>();
}
return boost::algorithm::to_lower_copy(
cfg.at(key).get<std::string>()) == "true";
}
return cfg.at(key).get<T>();
} catch (std::exception& e) {
PanicInfo(ErrorCode::UnexpectedError,
"get value from config for key {} failed, error: {}",
key,
e.what());
}
PanicInfo(ErrorCode::UnexpectedError,
"config type error for key {}: {}",
key,
e.what());
} catch (const std::exception& e) {
PanicInfo(ErrorCode::UnexpectedError,
"Unexpected error for key {}: {}",
key,
e.what());
}
return std::nullopt;
}

25
internal/core/unittest/test_c_api.cpp
View File

@ -4617,7 +4617,7 @@ TEST(CApiTest, TestGetValueFromConfig) {
GetValueFromConfig<std::string>(cfg, "d");
} catch (const std::exception& e) {
std::cout << e.what() << std::endl;
ASSERT_EQ(std::string(e.what()).find("get value from config for key") !=
ASSERT_EQ(std::string(e.what()).find("config type error for key") !=
std::string::npos,
true);
}
@ -4625,3 +4625,26 @@ TEST(CApiTest, TestGetValueFromConfig) {
auto e_value = GetValueFromConfig<std::string>(cfg, "e");
ASSERT_FALSE(e_value.has_value());
}
TEST(CApiTest, TestGetValueFromConfig_Without_Type_Check) {
nlohmann::json cfg = nlohmann::json::parse(
R"({"a" : 100, "b" : true, "c" : "true", "d" : 1.234, "e" : "1.234", "f" : null})");
SetDefaultConfigParamTypeCheck(false);
auto a_value = GetValueFromConfig<int64_t>(cfg, "a");
ASSERT_EQ(a_value.value(), 100);
std::cout << "a_value: " << a_value.value() << std::endl;
auto b_value = GetValueFromConfig<bool>(cfg, "b");
ASSERT_TRUE(b_value.value());
std::cout << "b_value: " << b_value.value() << std::endl;
auto c_value = GetValueFromConfig<bool>(cfg, "c");
ASSERT_TRUE(c_value.value());
std::cout << "c_value: " << c_value.value() << std::endl;
auto d_value = GetValueFromConfig<double>(cfg, "d");
ASSERT_NEAR(d_value.value(), 1.234, 0.001);
std::cout << "d_value: " << d_value.value() << std::endl;
auto e_value = GetValueFromConfig<double>(cfg, "e");
ASSERT_FALSE(e_value.has_value());
auto f_value = GetValueFromConfig<bool>(cfg, "f");
ASSERT_FALSE(f_value.has_value());
}

3
internal/querynodev2/server.go
View File

@ -259,6 +259,9 @@ func (node *QueryNode) InitSegcore() error {
cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)
cEnableConfigParamTypeCheck := C.bool(paramtable.Get().CommonCfg.EnableConfigParamTypeCheck.GetAsBool())
C.InitDefaultConfigParamTypeCheck(cEnableConfigParamTypeCheck)
localDataRootPath := filepath.Join(paramtable.Get().LocalStorageCfg.Path.GetValue(), typeutil.QueryNodeRole)
initcore.InitLocalChunkManager(localDataRootPath)

11
pkg/util/paramtable/component_param.go
View File

@ -302,6 +302,8 @@ type commonConfig struct {
EnabledOptimizeExpr ParamItem `refreshable:"true"`
EnabledJSONKeyStats ParamItem `refreshable:"true"`
EnabledGrowingSegmentJSONKeyStats ParamItem `refreshable:"true"`
EnableConfigParamTypeCheck ParamItem `refreshable:"true"`
}
func (p *commonConfig) init(base *BaseTable) {
@ -1030,6 +1032,15 @@ This helps Milvus-CDC synchronize incremental data`,
Export: true,
}
p.EnabledGrowingSegmentJSONKeyStats.Init(base.mgr)
p.EnableConfigParamTypeCheck = ParamItem{
Key: "common.enableConfigParamTypeCheck",
Version: "2.5.5",
DefaultValue: "true",
Doc: "Indicates whether to enable config param type check",
Export: true,
}
p.EnableConfigParamTypeCheck.Init(base.mgr)
}
type gpuConfig struct {