admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove hardcode max shards num (#22004) (#22011)

Browse Source 
Signed-off-by: longjiquan <jiquan.long@zilliz.com>
This commit is contained in:
Jiquan Long 2023-02-07 10:11:54 +08:00 committed by GitHub
parent a10620c5e8
commit 2feed29231
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 43 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
configs/milvus.yaml
View File

@ -159,7 +159,8 @@ proxy:
maxNameLength: 255 # Maximum length of name for a collection or alias
maxFieldNum: 256 # Maximum number of fields in a collection
maxDimension: 32768 # Maximum dimension of a vector
maxShardNum: 256 # Maximum number of shards in a collection
# It's strongly DISCOURAGED to set `maxShardNum` > 64.
maxShardNum: 64 # Maximum number of shards in a collection
maxTaskNum: 1024 # max task number of proxy task queue
# please adjust in embedded Milvus: false
ginLogging: true # Whether to produce gin logs.

13
internal/rootcoord/create_collection_task.go
View File

@ -51,9 +51,18 @@ func (t *createCollectionTask) validate() error {
return err
}
if t.Req.GetShardsNum() >= maxShardNum {
return fmt.Errorf("shard num (%d) exceeds limit (%d)", t.Req.GetShardsNum(), maxShardNum)
shardsNum := int64(t.Req.GetShardsNum())
cfgMaxShardNum := Params.RootCoordCfg.DmlChannelNum
if shardsNum >= cfgMaxShardNum {
return fmt.Errorf("shard num (%d) exceeds max configuration (%d)", shardsNum, cfgMaxShardNum)
}
cfgShardLimit := int64(Params.ProxyCfg.MaxShardNum)
if shardsNum > cfgShardLimit {
return fmt.Errorf("shard num (%d) exceeds system limit (%d)", shardsNum, cfgShardLimit)
}
return nil
}

32
internal/rootcoord/create_collection_task_test.go
View File

@ -40,11 +40,39 @@ func Test_createCollectionTask_validate(t *testing.T) {
assert.Error(t, err)
})
t.Run("shard num exceeds limit", func(t *testing.T) {
t.Run("shard num exceeds configuration", func(t *testing.T) {
cfgMaxShardNum := Params.RootCoordCfg.DmlChannelNum
restoreCfg := func() { Params.RootCoordCfg.DmlChannelNum = cfgMaxShardNum }
defer restoreCfg()
Params.RootCoordCfg.DmlChannelNum = 1
task := createCollectionTask{
Req: &milvuspb.CreateCollectionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_CreateCollection},
ShardsNum: maxShardNum + 1,
ShardsNum: 2,
},
}
err := task.validate()
assert.Error(t, err)
})
t.Run("shard num exceeds limit", func(t *testing.T) {
cfgMaxShardNum := Params.RootCoordCfg.DmlChannelNum
cfgShardLimit := Params.ProxyCfg.MaxShardNum
restoreCfg := func() {
Params.RootCoordCfg.DmlChannelNum = cfgMaxShardNum
Params.ProxyCfg.MaxShardNum = cfgShardLimit
}
defer restoreCfg()
Params.RootCoordCfg.DmlChannelNum = 100
Params.ProxyCfg.MaxShardNum = 4
task := createCollectionTask{
Req: &milvuspb.CreateCollectionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_CreateCollection},
ShardsNum: 8,
},
}
err := task.validate()