From 70246f82fda201da44f08aab2dba1e386b74ca47 Mon Sep 17 00:00:00 2001 From: Jiquan Long Date: Tue, 19 Jul 2022 13:50:28 +0800 Subject: [PATCH] Reduce chunk row size (#18320) Signed-off-by: longjiquan --- configs/embedded-milvus.yaml | 2 +- configs/milvus.yaml | 4 ++-- internal/util/paramtable/component_param.go | 2 +- internal/util/paramtable/component_param_test.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/embedded-milvus.yaml b/configs/embedded-milvus.yaml index d9c60ac95e..dfea43ce9f 100644 --- a/configs/embedded-milvus.yaml +++ b/configs/embedded-milvus.yaml @@ -170,7 +170,7 @@ queryNode: maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph # Segcore will divide a segment into multiple chunks. segcore: - chunkRows: 32768 # The number of vectors in a chunk. + chunkRows: 1024 # The number of vectors in a chunk. cache: enabled: true diff --git a/configs/milvus.yaml b/configs/milvus.yaml index a50fc8276f..931707c487 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -173,11 +173,11 @@ queryNode: maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph # Segcore will divide a segment into multiple chunks to enbale small index segcore: - chunkRows: 32768 # The number of vectors in a chunk. + chunkRows: 1024 # The number of vectors in a chunk. # Note: we have disabled segment small index since @2022.05.12. So below related configurations won't work. # We won't create small index for growing segments and search on these segments will directly use bruteforce scan. smallIndex: - nlist: 256 # small index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 + nlist: 128 # small index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist cache: enabled: true diff --git a/internal/util/paramtable/component_param.go b/internal/util/paramtable/component_param.go index 1a00b1e39c..f252348c64 100644 --- a/internal/util/paramtable/component_param.go +++ b/internal/util/paramtable/component_param.go @@ -791,7 +791,7 @@ func (p *queryNodeConfig) initFlowGraphMaxParallelism() { } func (p *queryNodeConfig) initSmallIndexParams() { - p.ChunkRows = p.Base.ParseInt64WithDefault("queryNode.segcore.chunkRows", 32768) + p.ChunkRows = p.Base.ParseInt64WithDefault("queryNode.segcore.chunkRows", 1024) if p.ChunkRows < 1024 { log.Warn("chunk rows can not be less than 1024, force set to 1024", zap.Any("current", p.ChunkRows)) p.ChunkRows = 1024 diff --git a/internal/util/paramtable/component_param_test.go b/internal/util/paramtable/component_param_test.go index 68b7b10b67..aa5e3a6aa1 100644 --- a/internal/util/paramtable/component_param_test.go +++ b/internal/util/paramtable/component_param_test.go @@ -231,10 +231,10 @@ func TestComponentParam(t *testing.T) { // test query side config chunkRows := Params.ChunkRows - assert.Equal(t, int64(32768), chunkRows) + assert.Equal(t, int64(1024), chunkRows) nlist := Params.SmallIndexNlist - assert.Equal(t, int64(256), nlist) + assert.Equal(t, int64(128), nlist) nprobe := Params.SmallIndexNProbe assert.Equal(t, int64(16), nprobe)