diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 0281012043..cbc3a0fa5b 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -506,7 +506,6 @@ indexCoord: nodeID: 0 segment: minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed - enabledJsonKeyStats: true # Indicates whether to enable JSON key stats indexNode: scheduler: @@ -894,6 +893,7 @@ common: localRPCEnabled: false # enable local rpc for internal communication when mix or standalone mode. sync: taskPoolReleaseTimeoutSeconds: 60 # The maximum time to wait for the task to finish and release resources in the pool + enabledJsonKeyStats: false # Indicates whether to enable JSON key stats # QuotaConfig, configurations of Milvus quota and limits. # By default, we enable: diff --git a/internal/datacoord/job_manager.go b/internal/datacoord/job_manager.go index 7c6766ab19..9eebe5e0e2 100644 --- a/internal/datacoord/job_manager.go +++ b/internal/datacoord/job_manager.go @@ -200,7 +200,7 @@ func (jm *statsJobManager) triggerJsonKeyIndexStatsTask() { needTriggerFieldIDs := make([]UniqueID, 0) for _, field := range collection.Schema.GetFields() { h := typeutil.CreateFieldSchemaHelper(field) - if h.EnableJSONKeyIndex() && Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool() { + if h.EnableJSONKeyIndex() && Params.CommonCfg.EnabledJSONKeyStats.GetAsBool() { needTriggerFieldIDs = append(needTriggerFieldIDs, field.GetFieldID()) } } diff --git a/internal/datacoord/job_manager_test.go b/internal/datacoord/job_manager_test.go index c30be4b53b..5b94f3bdf1 100644 --- a/internal/datacoord/job_manager_test.go +++ b/internal/datacoord/job_manager_test.go @@ -123,5 +123,5 @@ func (s *jobManagerSuite) TestJobManager_triggerStatsTaskLoop() { jm.loopWg.Wait() - s.Equal(3, jm.scheduler.pendingTasks.TaskCount()) + s.Equal(2, jm.scheduler.pendingTasks.TaskCount()) } diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index d881aa1ef4..960a25aa8c 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -205,7 +205,7 @@ func (i *IndexNode) initSegcore() { cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64()) C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval) - cJSONIndexEnabled := C.bool(Params.DataCoordCfg.EnabledJSONKeyStats.GetAsBool()) + cJSONIndexEnabled := C.bool(Params.CommonCfg.EnabledJSONKeyStats.GetAsBool()) C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled) } diff --git a/internal/querycoordv2/checkers/stats_checker.go b/internal/querycoordv2/checkers/stats_checker.go index c8dc5f7021..f23dd99e0e 100644 --- a/internal/querycoordv2/checkers/stats_checker.go +++ b/internal/querycoordv2/checkers/stats_checker.go @@ -32,6 +32,7 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/proto/datapb" "github.com/milvus-io/milvus/pkg/proto/querypb" + "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -142,7 +143,7 @@ func (c *StatsChecker) checkSegment(segment *meta.Segment, resp *milvuspb.Descri var result []int64 for _, field := range resp.GetSchema().GetFields() { h := typeutil.CreateFieldSchemaHelper(field) - if h.EnableJSONKeyIndex() { + if h.EnableJSONKeyIndex() && paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool() { exists := false for i := 0; i < len(segment.JSONIndexField); i++ { if segment.JSONIndexField[i] == field.FieldID { diff --git a/internal/querycoordv2/checkers/stats_checker_test.go b/internal/querycoordv2/checkers/stats_checker_test.go index 948867b2ba..43f3b5cd35 100644 --- a/internal/querycoordv2/checkers/stats_checker_test.go +++ b/internal/querycoordv2/checkers/stats_checker_test.go @@ -94,7 +94,8 @@ func (suite *StatsCheckerSuite) TearDownTest() { func (suite *StatsCheckerSuite) TestLoadJsonIndex() { checker := suite.checker ctx := context.Background() - + paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true") + defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key) // meta coll := utils.CreateTestCollection(1, 1) coll.FieldIndexID = map[int64]int64{101: 1000} @@ -155,7 +156,8 @@ func (suite *StatsCheckerSuite) TestLoadJsonIndex() { func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() { checker := suite.checker ctx := context.Background() - + paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true") + defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key) // meta coll := utils.CreateTestCollection(1, 1) coll.FieldIndexID = map[int64]int64{101: 1000} @@ -198,7 +200,8 @@ func (suite *StatsCheckerSuite) TestJsonIndexNotMatch() { func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() { checker := suite.checker ctx := context.Background() - + paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true") + defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key) // meta coll := utils.CreateTestCollection(1, 1) coll.FieldIndexID = map[int64]int64{101: 1000} @@ -231,7 +234,8 @@ func (suite *StatsCheckerSuite) TestDescribeCollectionFailed() { func (suite *StatsCheckerSuite) TestCreateNewJsonIndex() { checker := suite.checker ctx := context.Background() - + paramtable.Get().Save(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key, "true") + defer paramtable.Get().Reset(paramtable.Get().CommonCfg.EnabledJSONKeyStats.Key) // meta coll := utils.CreateTestCollection(1, 1) coll.FieldIndexID = map[int64]int64{101: 1000} diff --git a/internal/querynodev2/server.go b/internal/querynodev2/server.go index 0816535893..ba18c95e39 100644 --- a/internal/querynodev2/server.go +++ b/internal/querynodev2/server.go @@ -239,6 +239,9 @@ func (node *QueryNode) InitSegcore() error { cJSONIndexCommitInterval := C.int64_t(paramtable.Get().QueryNodeCfg.JSONIndexCommitInterval.GetAsInt64()) C.InitDefaultJSONKeyIndexCommitInterval(cJSONIndexCommitInterval) + cJSONIndexEnabled := C.bool(paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool()) + C.InitDefaultJSONKeyIndexEnable(cJSONIndexEnabled) + cGpuMemoryPoolInitSize := C.uint32_t(paramtable.Get().GpuConfig.InitSize.GetAsUint32()) cGpuMemoryPoolMaxSize := C.uint32_t(paramtable.Get().GpuConfig.MaxSize.GetAsUint32()) C.SegcoreSetKnowhereGpuMemoryPoolSize(cGpuMemoryPoolInitSize, cGpuMemoryPoolMaxSize) diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 88877657fa..a9e89b224c 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -296,6 +296,8 @@ type commonConfig struct { LocalRPCEnabled ParamItem `refreshable:"false"` SyncTaskPoolReleaseTimeoutSeconds ParamItem `refreshable:"true"` + + EnabledJSONKeyStats ParamItem `refreshable:"true"` } func (p *commonConfig) init(base *BaseTable) { @@ -998,6 +1000,15 @@ This helps Milvus-CDC synchronize incremental data`, Export: true, } p.SyncTaskPoolReleaseTimeoutSeconds.Init(base.mgr) + + p.EnabledJSONKeyStats = ParamItem{ + Key: "common.enabledJsonKeyStats", + Version: "2.5.5", + DefaultValue: "false", + Doc: "Indicates whether to enable JSON key stats", + Export: true, + } + p.EnabledJSONKeyStats.Init(base.mgr) } type gpuConfig struct { @@ -3488,8 +3499,6 @@ type dataCoordConfig struct { MinSegmentNumRowsToEnableIndex ParamItem `refreshable:"true"` BrokerTimeout ParamItem `refreshable:"false"` - EnabledJSONKeyStats ParamItem `refreshable:"true"` - // auto balance channel on datanode AutoBalance ParamItem `refreshable:"true"` CheckAutoBalanceConfigInterval ParamItem `refreshable:"false"` @@ -4153,15 +4162,6 @@ During compaction, the size of segment # of rows is able to exceed segment max # } p.MinSegmentNumRowsToEnableIndex.Init(base.mgr) - p.EnabledJSONKeyStats = ParamItem{ - Key: "indexCoord.enabledJsonKeyStats", - Version: "2.0.0", - DefaultValue: "true", - Doc: "Indicates whether to enable JSON key stats", - Export: true, - } - p.EnabledJSONKeyStats.Init(base.mgr) - p.BindIndexNodeMode = ParamItem{ Key: "indexCoord.bindIndexNodeMode.enable", Version: "2.0.0",