admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Refine segcore param change callback (#43838)

Browse Source 
Related to #43230

This PR
- Move segcore setup function to `initcore` package to remove cgo
dependency from pkg
- Register core callback only for components depends on segcore
- Rectify `UpdateLogLevel` implementation

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2025-08-13 19:31:44 +08:00 committed by GitHub
parent 9f9ed14ddb
commit f032044125
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 130 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
internal/core/src/common/Common.cpp
View File

@ -15,6 +15,7 @@
// limitations under the License.
#include "common/Common.h"
#include "gflags/gflags.h"
#include "log/Log.h"
namespace milvus {
@ -76,11 +77,20 @@ void
SetLogLevel(const char* level) {
LOG_INFO("set log level: {}", level);
if (strcmp(level, "debug") == 0) {
FLAGS_v = 5;
gflags::SetCommandLineOption("minloglevel", "0");
gflags::SetCommandLineOption("v", "5");
} else if (strcmp(level, "trace") == 0) {
FLAGS_v = 6;
gflags::SetCommandLineOption("minloglevel", "0");
gflags::SetCommandLineOption("v", "6");
} else {
FLAGS_v = 4;
gflags::SetCommandLineOption("v", "4");
if (strcmp(level, "info") == 0) {
gflags::SetCommandLineOption("minloglevel", "0");
} else if (strcmp(level, "warn") == 0) {
gflags::SetCommandLineOption("minloglevel", "1");
} else if (strcmp(level, "error") == 0) {
gflags::SetCommandLineOption("minloglevel", "2");
}
}
}

6
internal/datanode/index/init_segcore.go
View File

@ -43,6 +43,9 @@ func InitSegcore() {
C.IndexBuilderInit(cGlogConf)
C.free(unsafe.Pointer(cGlogConf))
// update log level based on current setup
initcore.UpdateLogLevel(paramtable.Get().LogCfg.Level.GetValue())
// override index builder SIMD type
cSimdType := C.CString(paramtable.Get().CommonCfg.SimdType.GetValue())
C.IndexBuilderSetSimdType(cSimdType)
@ -78,6 +81,9 @@ func InitSegcore() {
cGpuMemoryPoolInitSize := C.uint32_t(paramtable.Get().GpuConfig.InitSize.GetAsUint32())
cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32())
C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize)
// init paramtable change callback for core related config
initcore.SetupCoreConfigChangelCallback()
}
func CloseSegcore() {

6
internal/querynodev2/server.go
View File

@ -231,6 +231,9 @@ func (node *QueryNode) InitSegcore() error {
C.SegcoreInit(cGlogConf)
C.free(unsafe.Pointer(cGlogConf))
// update log level based on current setup
initcore.UpdateLogLevel(paramtable.Get().LogCfg.Level.GetValue())
// override segcore chunk size
cChunkRows := C.int64_t(paramtable.Get().QueryNodeCfg.ChunkRows.GetAsInt64())
C.SegcoreSetChunkRows(cChunkRows)
@ -412,6 +415,9 @@ func (node *QueryNode) InitSegcore() error {
initcore.InitTraceConfig(paramtable.Get())
C.InitExecExpressionFunctionFactory()
// init paramtable change callback for core related config
initcore.SetupCoreConfigChangelCallback()
return nil
}

3
internal/streamingnode/server/server.go
View File

@ -47,6 +47,9 @@ func (s *Server) init() {
if err := initcore.InitStorageV2FileSystem(paramtable.Get()); err != nil {
panic(fmt.Sprintf("unrecoverable error happens at init storage v2 file system, %+v", err))
}
// init paramtable change callback for core related config
initcore.SetupCoreConfigChangelCallback()
}
// Stop stops the streamingnode server.

94
internal/util/initcore/init_core.go
View File

@ -29,9 +29,12 @@ package initcore
import "C"
import (
"context"
"encoding/base64"
"fmt"
"path"
"strconv"
"sync"
"time"
"unsafe"
@ -281,6 +284,97 @@ func InitFileWriterConfig(params *paramtable.ComponentParam) error {
return HandleCStatus(&status, "InitFileWriterConfig failed")
}
var coreParamCallbackInitOnce sync.Once
func SetupCoreConfigChangelCallback() {
coreParamCallbackInitOnce.Do(func() {
paramtable.Get().CommonCfg.IndexSliceSize.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
size, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateIndexSliceSize(size)
return nil
})
paramtable.Get().CommonCfg.HighPriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateHighPriorityThreadCoreCoefficient(coefficient)
return nil
})
paramtable.Get().CommonCfg.MiddlePriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateMiddlePriorityThreadCoreCoefficient(coefficient)
return nil
})
paramtable.Get().CommonCfg.LowPriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateLowPriorityThreadCoreCoefficient(coefficient)
return nil
})
paramtable.Get().CommonCfg.EnabledOptimizeExpr.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultOptimizeExprEnable(enable)
return nil
})
paramtable.Get().CommonCfg.EnabledGrowingSegmentJSONKeyStats.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultGrowingJSONKeyStatsEnable(enable)
return nil
})
paramtable.Get().CommonCfg.EnableConfigParamTypeCheck.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultConfigParamTypeCheck(enable)
return nil
})
paramtable.Get().LogCfg.Level.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
return UpdateLogLevel(newValue)
})
paramtable.Get().QueryNodeCfg.ExprEvalBatchSize.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
size, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateDefaultExprEvalBatchSize(size)
return nil
})
paramtable.Get().QueryNodeCfg.JSONKeyStatsCommitInterval.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
interval, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateDefaultJSONKeyStatsCommitInterval(interval)
return nil
})
})
}
func InitInterminIndexConfig(params *paramtable.ComponentParam) error {
enableInterminIndex := C.bool(params.QueryNodeCfg.EnableInterminSegmentIndex.GetAsBool())
C.SegcoreSetEnableInterminSegmentIndex(enableInterminIndex)

10
pkg/util/paramtable/util.go → internal/util/initcore/util.go
View File

@ -14,7 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package paramtable
package initcore
/*
#cgo pkg-config: milvus_core
@ -25,9 +25,15 @@ package paramtable
#include "common/init_c.h"
*/
import "C"
import "unsafe"
import (
"strings"
"unsafe"
)
func UpdateLogLevel(level string) error {
// always use lower case
level = strings.ToLower(level)
cvalue := C.CString(level)
C.SetLogLevel(cvalue)
C.free(unsafe.Pointer(cvalue))

76
pkg/util/paramtable/component_param.go
View File

@ -17,7 +17,6 @@
package paramtable
import (
"context"
"fmt"
"os"
"path"
@ -531,14 +530,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
Export: true,
}
p.IndexSliceSize.Init(base.mgr)
p.IndexSliceSize.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
size, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateIndexSliceSize(size)
return nil
})
p.EnableMaterializedView = ParamItem{
Key: "common.materializedView.enabled",
@ -640,14 +631,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
Export: true,
}
p.HighPriorityThreadCoreCoefficient.Init(base.mgr)
p.HighPriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateHighPriorityThreadCoreCoefficient(coefficient)
return nil
})
p.MiddlePriorityThreadCoreCoefficient = ParamItem{
Key: "common.threadCoreCoefficient.middlePriority",
@ -658,14 +641,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
Export: true,
}
p.MiddlePriorityThreadCoreCoefficient.Init(base.mgr)
p.MiddlePriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateMiddlePriorityThreadCoreCoefficient(coefficient)
return nil
})
p.LowPriorityThreadCoreCoefficient = ParamItem{
Key: "common.threadCoreCoefficient.lowPriority",
@ -676,14 +651,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
Export: true,
}
p.LowPriorityThreadCoreCoefficient.Init(base.mgr)
p.LowPriorityThreadCoreCoefficient.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
coefficient, err := strconv.ParseFloat(newValue, 64)
if err != nil {
return err
}
UpdateLowPriorityThreadCoreCoefficient(coefficient)
return nil
})
p.DiskWriteMode = ParamItem{
Key: "common.diskWriteMode",
@ -1097,14 +1064,6 @@ This helps Milvus-CDC synchronize incremental data`,
Export: true,
}
p.EnabledOptimizeExpr.Init(base.mgr)
p.EnabledOptimizeExpr.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultOptimizeExprEnable(enable)
return nil
})
p.EnabledJSONKeyStats = ParamItem{
Key: "common.enabledJSONKeyStats",
@ -1123,14 +1082,6 @@ This helps Milvus-CDC synchronize incremental data`,
Export: true,
}
p.EnabledGrowingSegmentJSONKeyStats.Init(base.mgr)
p.EnabledGrowingSegmentJSONKeyStats.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultGrowingJSONKeyStatsEnable(enable)
return nil
})
p.EnableConfigParamTypeCheck = ParamItem{
Key: "common.enableConfigParamTypeCheck",
@ -1140,14 +1091,6 @@ This helps Milvus-CDC synchronize incremental data`,
Export: true,
}
p.EnableConfigParamTypeCheck.Init(base.mgr)
p.EnableConfigParamTypeCheck.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
enable, err := strconv.ParseBool(newValue)
if err != nil {
return err
}
UpdateDefaultConfigParamTypeCheck(enable)
return nil
})
}
type gpuConfig struct {
@ -1388,9 +1331,6 @@ It is recommended to use debug level under test and development environments, an
Export: true,
}
l.Level.Init(base.mgr)
l.Level.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
return UpdateLogLevel(newValue)
})
l.RootPath = ParamItem{
Key: "log.file.rootPath",
@ -3943,14 +3883,6 @@ user-task-polling:
Doc: "expr eval batch size for getnext interface",
}
p.ExprEvalBatchSize.Init(base.mgr)
p.ExprEvalBatchSize.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
size, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateDefaultExprEvalBatchSize(size)
return nil
})
p.JSONKeyStatsCommitInterval = ParamItem{
Key: "queryNode.segcore.jsonKeyStatsCommitInterval",
@ -3960,14 +3892,6 @@ user-task-polling:
Export: true,
}
p.JSONKeyStatsCommitInterval.Init(base.mgr)
p.JSONKeyStatsCommitInterval.RegisterCallback(func(ctx context.Context, key, oldValue, newValue string) error {
interval, err := strconv.Atoi(newValue)
if err != nil {
return err
}
UpdateDefaultJSONKeyStatsCommitInterval(interval)
return nil
})
p.CleanExcludeSegInterval = ParamItem{
Key: "queryCoord.cleanExcludeSegmentInterval",