diff --git a/internal/core/src/common/Array.h b/internal/core/src/common/Array.h index 09f0d2a381..26b3160426 100644 --- a/internal/core/src/common/Array.h +++ b/internal/core/src/common/Array.h @@ -296,11 +296,12 @@ class Array { return offsets_ptr_.get(); } - ScalarFieldProto - output_data() const { - ScalarFieldProto data_array; + void + output_data(ScalarFieldProto& data_array) const { switch (element_type_) { case DataType::BOOL: { + data_array.mutable_bool_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_bool_data()->add_data(element); @@ -310,6 +311,7 @@ class Array { case DataType::INT8: case DataType::INT16: case DataType::INT32: { + data_array.mutable_int_data()->mutable_data()->Reserve(length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_int_data()->add_data(element); @@ -317,6 +319,8 @@ class Array { break; } case DataType::INT64: { + data_array.mutable_long_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_long_data()->add_data(element); @@ -325,13 +329,18 @@ class Array { } case DataType::STRING: case DataType::VARCHAR: { + data_array.mutable_string_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { - auto element = get_data(j); - data_array.mutable_string_data()->add_data(element); + auto element = get_data(j); + data_array.mutable_string_data()->add_data(element.data(), + element.size()); } break; } case DataType::FLOAT: { + data_array.mutable_float_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_float_data()->add_data(element); @@ -339,6 +348,8 @@ class Array { break; } case DataType::DOUBLE: { + data_array.mutable_double_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_double_data()->add_data(element); @@ -346,9 +357,12 @@ class Array { break; } case DataType::GEOMETRY: { + data_array.mutable_geometry_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { - auto element = get_data(j); - data_array.mutable_geometry_data()->add_data(element); + auto element = get_data(j); + data_array.mutable_geometry_data()->add_data( + element.data(), element.size()); } break; } @@ -356,6 +370,12 @@ class Array { // empty array } } + } + + ScalarFieldProto + output_data() const { + ScalarFieldProto data_array; + output_data(data_array); return data_array; } @@ -541,11 +561,12 @@ class ArrayView { return reinterpret_cast(data_)[index]; } - ScalarFieldProto - output_data() const { - ScalarFieldProto data_array; + void + output_data(ScalarFieldProto& data_array) const { switch (element_type_) { case DataType::BOOL: { + data_array.mutable_bool_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_bool_data()->add_data(element); @@ -555,6 +576,7 @@ class ArrayView { case DataType::INT8: case DataType::INT16: case DataType::INT32: { + data_array.mutable_int_data()->mutable_data()->Reserve(length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_int_data()->add_data(element); @@ -562,6 +584,8 @@ class ArrayView { break; } case DataType::INT64: { + data_array.mutable_long_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_long_data()->add_data(element); @@ -570,13 +594,18 @@ class ArrayView { } case DataType::STRING: case DataType::VARCHAR: { + data_array.mutable_string_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { - auto element = get_data(j); - data_array.mutable_string_data()->add_data(element); + auto element = get_data(j); + data_array.mutable_string_data()->add_data(element.data(), + element.size()); } break; } case DataType::FLOAT: { + data_array.mutable_float_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_float_data()->add_data(element); @@ -584,6 +613,8 @@ class ArrayView { break; } case DataType::DOUBLE: { + data_array.mutable_double_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { auto element = get_data(j); data_array.mutable_double_data()->add_data(element); @@ -591,9 +622,12 @@ class ArrayView { break; } case DataType::GEOMETRY: { + data_array.mutable_geometry_data()->mutable_data()->Reserve( + length_); for (int j = 0; j < length_; ++j) { - auto element = get_data(j); - data_array.mutable_geometry_data()->add_data(element); + auto element = get_data(j); + data_array.mutable_geometry_data()->add_data( + element.data(), element.size()); } break; } @@ -601,6 +635,12 @@ class ArrayView { // empty array } } + } + + ScalarFieldProto + output_data() const { + ScalarFieldProto data_array; + output_data(data_array); return data_array; } diff --git a/internal/core/src/mmap/ChunkedColumn.h b/internal/core/src/mmap/ChunkedColumn.h index e0e9c7c3d6..17a8dd0c59 100644 --- a/internal/core/src/mmap/ChunkedColumn.h +++ b/internal/core/src/mmap/ChunkedColumn.h @@ -629,16 +629,15 @@ class ChunkedArrayColumn : public ChunkedColumnBase { void BulkArrayAt(milvus::OpContext* op_ctx, - std::function fn, + std::function fn, const int64_t* offsets, int64_t count) const override { auto [cids, offsets_in_chunk] = ToChunkIdAndOffset(offsets, count); auto ca = SemiInlineGet(slot_->PinCells(op_ctx, cids)); for (int64_t i = 0; i < count; i++) { - auto array = static_cast(ca->get_cell_of(cids[i])) - ->View(offsets_in_chunk[i]) - .output_data(); - fn(std::move(array), i); + auto view = static_cast(ca->get_cell_of(cids[i])) + ->View(offsets_in_chunk[i]); + fn(view, i); } } diff --git a/internal/core/src/mmap/ChunkedColumnGroup.h b/internal/core/src/mmap/ChunkedColumnGroup.h index 582b68c08d..08c56498d2 100644 --- a/internal/core/src/mmap/ChunkedColumnGroup.h +++ b/internal/core/src/mmap/ChunkedColumnGroup.h @@ -626,7 +626,7 @@ class ProxyChunkColumn : public ChunkedColumnInterface { void BulkArrayAt(milvus::OpContext* op_ctx, - std::function fn, + std::function fn, const int64_t* offsets, int64_t count) const override { if (!IsChunkedArrayColumnDataType(data_type_)) { @@ -639,10 +639,9 @@ class ProxyChunkColumn : public ChunkedColumnInterface { for (int64_t i = 0; i < count; i++) { auto* group_chunk = ca->get_cell_of(cids[i]); auto chunk = group_chunk->GetChunk(field_id_); - auto array = static_cast(chunk.get()) - ->View(offsets_in_chunk[i]) - .output_data(); - fn(std::move(array), i); + auto view = static_cast(chunk.get()) + ->View(offsets_in_chunk[i]); + fn(view, i); } } diff --git a/internal/core/src/mmap/ChunkedColumnInterface.h b/internal/core/src/mmap/ChunkedColumnInterface.h index faf8dfb6bb..f9c79d4b70 100644 --- a/internal/core/src/mmap/ChunkedColumnInterface.h +++ b/internal/core/src/mmap/ChunkedColumnInterface.h @@ -187,7 +187,7 @@ class ChunkedColumnInterface { virtual void BulkArrayAt(milvus::OpContext* op_ctx, - std::function fn, + std::function fn, const int64_t* offsets, int64_t count) const { ThrowInfo(ErrorCode::Unsupported, diff --git a/internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp b/internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp index fd1dfcd99b..add9eef5ca 100644 --- a/internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp +++ b/internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp @@ -1678,8 +1678,8 @@ ChunkedSegmentSealedImpl::bulk_subscript_array_impl( google::protobuf::RepeatedPtrField* dst) { column->BulkArrayAt( op_ctx, - [dst](ScalarFieldProto&& array, size_t i) { - dst->at(i) = std::move(array); + [dst](const ArrayView& view, size_t i) { + view.output_data(dst->at(i)); }, seg_offsets, count);