diff --git a/configs/advanced/query_node.yaml b/configs/advanced/query_node.yaml index 1f9a773132..e4e6a1ffd4 100644 --- a/configs/advanced/query_node.yaml +++ b/configs/advanced/query_node.yaml @@ -27,5 +27,5 @@ queryNode: recvBufSize: 64 segcore: - chunkSize: 32768 # 32M + chunkRows: 32768 simdType: auto # auto, avx512, avx2, sse diff --git a/internal/core/bench/bench_search.cpp b/internal/core/bench/bench_search.cpp index d4ad0b53e3..1ff8302461 100644 --- a/internal/core/bench/bench_search.cpp +++ b/internal/core/bench/bench_search.cpp @@ -68,9 +68,9 @@ Search_SmallIndex(benchmark::State& state) { }(); auto is_small_index = state.range(0); - auto chunk_size = state.range(1) * 1024; + auto chunk_rows = state.range(1) * 1024; auto segconf = SegcoreConfig::default_config(); - segconf.set_size_per_chunk(chunk_size); + segconf.set_chunk_rows(chunk_rows); auto segment = CreateGrowingSegment(schema, segconf); if (!is_small_index) { segment->disable_small_index(); diff --git a/internal/core/src/segcore/FieldIndexing.h b/internal/core/src/segcore/FieldIndexing.h index bb7f41c51e..6b99d9ff78 100644 --- a/internal/core/src/segcore/FieldIndexing.h +++ b/internal/core/src/segcore/FieldIndexing.h @@ -45,7 +45,7 @@ class FieldIndexing { int64_t get_size_per_chunk() const { - return segcore_config_.get_size_per_chunk(); + return segcore_config_.get_chunk_rows(); } virtual knowhere::Index* diff --git a/internal/core/src/segcore/SegcoreConfig.cpp b/internal/core/src/segcore/SegcoreConfig.cpp index 68b9da6e8b..ed8be1b75a 100644 --- a/internal/core/src/segcore/SegcoreConfig.cpp +++ b/internal/core/src/segcore/SegcoreConfig.cpp @@ -48,8 +48,8 @@ SegcoreConfig::parse_from(const std::string& config_path) { YAML::Node top_config = YAML::LoadFile(config_path); Assert(top_config.IsMap()); auto seg_config = subnode(top_config, "segcore"); - auto chunk_size = subnode(seg_config, "chunk_size").as(); - this->size_per_chunk_ = chunk_size; + auto chunk_rows = subnode(seg_config, "chunk_rows").as(); + this->chunk_rows_ = chunk_rows; #if 0 auto index_list = subnode(seg_config, "small_index"); diff --git a/internal/core/src/segcore/SegcoreConfig.h b/internal/core/src/segcore/SegcoreConfig.h index 6ada01a01e..42701bc11d 100644 --- a/internal/core/src/segcore/SegcoreConfig.h +++ b/internal/core/src/segcore/SegcoreConfig.h @@ -55,13 +55,13 @@ class SegcoreConfig { } int64_t - get_size_per_chunk() const { - return size_per_chunk_; + get_chunk_rows() const { + return chunk_rows_; } void - set_size_per_chunk(int64_t size_per_chunk) { - size_per_chunk_ = size_per_chunk; + set_chunk_rows(int64_t chunk_rows) { + chunk_rows_ = chunk_rows; } void @@ -70,7 +70,7 @@ class SegcoreConfig { } private: - int64_t size_per_chunk_ = 32 * 1024; + int64_t chunk_rows_ = 32 * 1024; std::map table_; }; diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index f4de039617..80aa3ede1d 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -203,8 +203,8 @@ SegmentGrowingImpl::do_insert(int64_t reserved_begin, record_.ack_responder_.AddSegment(reserved_begin, reserved_begin + size); if (enable_small_index_) { - int64_t chunk_size = segcore_config_.get_size_per_chunk(); - indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / chunk_size, record_); + int64_t chunk_rows = segcore_config_.get_chunk_rows(); + indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / chunk_rows, record_); } } @@ -243,10 +243,10 @@ SegmentGrowingImpl::Delete(int64_t reserved_begin, int64_t SegmentGrowingImpl::GetMemoryUsageInBytes() const { int64_t total_bytes = 0; - auto size_per_chunk = segcore_config_.get_size_per_chunk(); - int64_t ins_n = upper_align(record_.reserved, size_per_chunk); + auto chunk_rows = segcore_config_.get_chunk_rows(); + int64_t ins_n = upper_align(record_.reserved, chunk_rows); total_bytes += ins_n * (schema_->get_total_sizeof() + 16 + 1); - int64_t del_n = upper_align(deleted_record_.reserved, size_per_chunk); + int64_t del_n = upper_align(deleted_record_.reserved, chunk_rows); total_bytes += del_n * (16 * 2); return total_bytes; } @@ -260,7 +260,7 @@ SegmentGrowingImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) int64_t SegmentGrowingImpl::num_chunk() const { auto size = get_insert_record().ack_responder_.GetAck(); - return upper_div(size, segcore_config_.get_size_per_chunk()); + return upper_div(size, segcore_config_.get_chunk_rows()); } void diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index c2baf26b87..235865e780 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -109,7 +109,7 @@ class SegmentGrowingImpl : public SegmentGrowing { int64_t size_per_chunk() const final { - return segcore_config_.get_size_per_chunk(); + return segcore_config_.get_chunk_rows(); } public: @@ -159,7 +159,7 @@ class SegmentGrowingImpl : public SegmentGrowing { explicit SegmentGrowingImpl(SchemaPtr schema, const SegcoreConfig& segcore_config) : segcore_config_(segcore_config), schema_(std::move(schema)), - record_(*schema_, segcore_config.get_size_per_chunk()), + record_(*schema_, segcore_config.get_chunk_rows()), indexing_record_(*schema_, segcore_config_) { } diff --git a/internal/core/src/segcore/segcore_init_c.cpp b/internal/core/src/segcore/segcore_init_c.cpp index af36fcb3df..79d2d9b620 100644 --- a/internal/core/src/segcore/segcore_init_c.cpp +++ b/internal/core/src/segcore/segcore_init_c.cpp @@ -37,10 +37,10 @@ SegcoreInit() { } extern "C" void -SegcoreSetChunkSize(const int64_t value) { +SegcoreSetChunkRows(const int64_t value) { milvus::segcore::SegcoreConfig& config = milvus::segcore::SegcoreConfig::default_config(); - config.set_size_per_chunk(value); - std::cout << "set config chunk_size: " << config.get_size_per_chunk() << std::endl; + config.set_chunk_rows(value); + std::cout << "set config chunk_size: " << config.get_chunk_rows() << std::endl; } extern "C" void diff --git a/internal/core/src/segcore/segcore_init_c.h b/internal/core/src/segcore/segcore_init_c.h index 985faefa8f..2678f4ab72 100644 --- a/internal/core/src/segcore/segcore_init_c.h +++ b/internal/core/src/segcore/segcore_init_c.h @@ -19,7 +19,7 @@ void SegcoreInit(); void -SegcoreSetChunkSize(const int64_t); +SegcoreSetChunkRows(const int64_t); void SegcoreSetSimdType(const char*); diff --git a/internal/core/unittest/test_init.cpp b/internal/core/unittest/test_init.cpp index e93cce6995..4bbc43df6e 100644 --- a/internal/core/unittest/test_init.cpp +++ b/internal/core/unittest/test_init.cpp @@ -20,6 +20,6 @@ TEST(Init, Naive) { using namespace milvus; using namespace milvus::segcore; SegcoreInit(); - SegcoreSetChunkSize(32768); + SegcoreSetChunkRows(32768); SegcoreSetSimdType("auto"); } diff --git a/internal/core/unittest/test_utils/test_segcore.yaml b/internal/core/unittest/test_utils/test_segcore.yaml index cdad64f22d..a0c5dfa611 100644 --- a/internal/core/unittest/test_utils/test_segcore.yaml +++ b/internal/core/unittest/test_utils/test_segcore.yaml @@ -1,5 +1,5 @@ segcore: - chunk_size: 32768 + chunk_rows: 32768 small_index: - metric_type: ["L2", "IP"] index_type: "IVF" diff --git a/internal/querynode/param_table.go b/internal/querynode/param_table.go index 6a444aa202..7510709696 100644 --- a/internal/querynode/param_table.go +++ b/internal/querynode/param_table.go @@ -69,7 +69,7 @@ type ParamTable struct { SliceIndex int // segcore - ChunkSize int64 + ChunkRows int64 SimdType string Log log.Config @@ -115,7 +115,7 @@ func (p *ParamTable) Init() { p.initStatsPublishInterval() p.initStatsChannelName() - p.initSegcoreChunkSize() + p.initSegcoreChunkRows() p.initSegcoreSimdType() p.initLogCfg() @@ -261,8 +261,8 @@ func (p *ParamTable) initStatsChannelName() { p.StatsChannelName = channels } -func (p *ParamTable) initSegcoreChunkSize() { - p.ChunkSize = p.ParseInt64("queryNode.segcore.chunkSize") +func (p *ParamTable) initSegcoreChunkRows() { + p.ChunkRows = p.ParseInt64("queryNode.segcore.chunkRows") } func (p *ParamTable) initSegcoreSimdType() { diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index 02cc25ffd9..b1988b8bc9 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -103,8 +103,8 @@ func (node *QueryNode) InitSegcore() { C.SegcoreInit() // override segcore chunk size - cChunkSize := C.int64_t(Params.ChunkSize) - C.SegcoreSetChunkSize(cChunkSize) + cChunkRows := C.int64_t(Params.ChunkRows) + C.SegcoreSetChunkRows(cChunkRows) // override segcore SIMD type cSimdType := C.CString(Params.SimdType)