diff --git a/internal/core/src/common/ChunkWriter.cpp b/internal/core/src/common/ChunkWriter.cpp index ddcb1af977..401726f136 100644 --- a/internal/core/src/common/ChunkWriter.cpp +++ b/internal/core/src/common/ChunkWriter.cpp @@ -250,66 +250,135 @@ ArrayChunkWriter::finish() { std::move(mmap_file_raii)); } -// 1. Deserialize VectorFieldProto (proto::schema::VectorField) from arrow::ArrayVector -// where VectorFieldProto is vector array and each element it self is a VectorFieldProto. -// 2. Transform this vector of VectorFieldProto to vector of our local representation of VectorArray. -// 3. the contents of these VectorArray are concatenated in some format in target_. -// See more details for the format in the comments of VectorArrayChunk. +// Read vector array data from arrow::ArrayVector and write to target_ void -VectorArrayChunkWriter::write(const arrow::ArrayVector& arrow_array_vec) { - auto size = 0; - std::vector vector_arrays; - vector_arrays.reserve(arrow_array_vec.size()); +VectorArrayChunkWriter::write(const arrow::ArrayVector& array_vec) { + size_t total_size = calculateTotalSize(array_vec); + row_nums_ = 0; - for (const auto& data : arrow_array_vec) { - auto array = std::dynamic_pointer_cast(data); - for (size_t i = 0; i < array->length(); i++) { - auto str = array->GetView(i); - VectorFieldProto vector_field; - vector_field.ParseFromArray(str.data(), str.size()); - auto arr = VectorArray(vector_field); - size += arr.byte_size(); - vector_arrays.push_back(std::move(arr)); - } - row_nums_ += array->length(); + for (const auto& array_data : array_vec) { + row_nums_ += array_data->length(); } - // offsets + lens - size += sizeof(uint32_t) * (row_nums_ * 2 + 1) + MMAP_ARRAY_PADDING; if (!file_path_.empty()) { target_ = std::make_shared(file_path_); } else { - target_ = std::make_shared(size); + target_ = std::make_shared(total_size); } - int offsets_num = row_nums_ + 1; - int len_num = row_nums_; - uint32_t offset_start_pos = - target_->tell() + sizeof(uint32_t) * (offsets_num + len_num); - std::vector offsets(offsets_num); - std::vector lens(len_num); - for (size_t i = 0; i < vector_arrays.size(); i++) { - auto& arr = vector_arrays[i]; - offsets[i] = offset_start_pos; - lens[i] = arr.length(); - offset_start_pos += arr.byte_size(); - } - if (offsets_num > 0) { - offsets[offsets_num - 1] = offset_start_pos; - } - - for (int i = 0; i < offsets.size(); i++) { - if (i == offsets.size() - 1) { - target_->write(&offsets[i], sizeof(uint32_t)); + switch (element_type_) { + case milvus::DataType::VECTOR_FLOAT: + writeFloatVectorArray(array_vec); break; + case milvus::DataType::VECTOR_BINARY: + ThrowInfo(NotImplemented, + "BinaryVector in VectorArray not implemented yet"); + case milvus::DataType::VECTOR_FLOAT16: + ThrowInfo(NotImplemented, + "Float16Vector in VectorArray not implemented yet"); + case milvus::DataType::VECTOR_BFLOAT16: + ThrowInfo(NotImplemented, + "BFloat16Vector in VectorArray not implemented yet"); + case milvus::DataType::VECTOR_INT8: + ThrowInfo(NotImplemented, + "Int8Vector in VectorArray not implemented yet"); + default: + ThrowInfo(NotImplemented, + "Unsupported element type in VectorArray: {}", + static_cast(element_type_)); + } +} + +void +VectorArrayChunkWriter::writeFloatVectorArray( + const arrow::ArrayVector& array_vec) { + std::vector offsets_lens; + std::vector float_data_ptrs; + std::vector data_sizes; + + uint32_t current_offset = + sizeof(uint32_t) * (row_nums_ * 2 + 1) + target_->tell(); + + for (const auto& array_data : array_vec) { + auto list_array = + std::static_pointer_cast(array_data); + auto float_values = + std::static_pointer_cast(list_array->values()); + const float* raw_floats = float_values->raw_values(); + const int32_t* list_offsets = list_array->raw_value_offsets(); + + // Generate offsets and lengths for each row + // Each list contains multiple float vectors which are flattened, so the float count + // in each list is vector count * dim. + for (int64_t i = 0; i < list_array->length(); i++) { + auto start_idx = list_offsets[i]; + auto end_idx = list_offsets[i + 1]; + auto vector_count = (end_idx - start_idx) / dim_; + auto byte_size = (end_idx - start_idx) * sizeof(float); + + offsets_lens.push_back(current_offset); + offsets_lens.push_back(static_cast(vector_count)); + + float_data_ptrs.push_back(raw_floats + start_idx); + data_sizes.push_back(byte_size); + + current_offset += byte_size; } - target_->write(&offsets[i], sizeof(uint32_t)); - target_->write(&lens[i], sizeof(uint32_t)); } - for (auto& arr : vector_arrays) { - target_->write(arr.data(), arr.byte_size()); + // Add final offset + offsets_lens.push_back(current_offset); + + // Write offset and length arrays + for (size_t i = 0; i < offsets_lens.size() - 1; i += 2) { + target_->write(&offsets_lens[i], sizeof(uint32_t)); // offset + target_->write(&offsets_lens[i + 1], sizeof(uint32_t)); // length } + target_->write(&offsets_lens.back(), sizeof(uint32_t)); // final offset + + for (size_t i = 0; i < float_data_ptrs.size(); i++) { + target_->write(float_data_ptrs[i], data_sizes[i]); + } +} + +size_t +VectorArrayChunkWriter::calculateTotalSize( + const arrow::ArrayVector& array_vec) { + size_t total_size = 0; + size_t total_rows = 0; + + // Calculate total size for vector data and count rows + for (const auto& array_data : array_vec) { + total_rows += array_data->length(); + auto list_array = + std::static_pointer_cast(array_data); + + switch (element_type_) { + case milvus::DataType::VECTOR_FLOAT: { + auto float_values = std::static_pointer_cast( + list_array->values()); + total_size += float_values->length() * sizeof(float); + break; + } + case milvus::DataType::VECTOR_BINARY: + case milvus::DataType::VECTOR_FLOAT16: + case milvus::DataType::VECTOR_BFLOAT16: + case milvus::DataType::VECTOR_INT8: + ThrowInfo(NotImplemented, + "Element type {} in VectorArray not implemented yet", + static_cast(element_type_)); + default: + ThrowInfo(DataTypeInvalid, + "Invalid element type {} for VectorArray", + static_cast(element_type_)); + } + } + + // Add space for offset and length arrays + total_size += sizeof(uint32_t) * (total_rows * 2 + 1 /* final offset */) + + MMAP_ARRAY_PADDING; + + return total_size; } std::unique_ptr diff --git a/internal/core/src/common/ChunkWriter.h b/internal/core/src/common/ChunkWriter.h index b5adb3889b..58aa3c21e2 100644 --- a/internal/core/src/common/ChunkWriter.h +++ b/internal/core/src/common/ChunkWriter.h @@ -245,12 +245,9 @@ class ArrayChunkWriter : public ChunkWriterBase { class VectorArrayChunkWriter : public ChunkWriterBase { public: - VectorArrayChunkWriter(int64_t dim, const milvus::DataType element_type) - : ChunkWriterBase(false), element_type_(element_type), dim_(dim) { - } VectorArrayChunkWriter(int64_t dim, const milvus::DataType element_type, - std::string file_path) + std::string file_path = "") : ChunkWriterBase(std::move(file_path), false), element_type_(element_type), dim_(dim) { @@ -263,6 +260,12 @@ class VectorArrayChunkWriter : public ChunkWriterBase { finish() override; private: + void + writeFloatVectorArray(const arrow::ArrayVector& array_vec); + + size_t + calculateTotalSize(const arrow::ArrayVector& array_vec); + const milvus::DataType element_type_; int64_t dim_; }; diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h index a97331fe3e..31354f6d6a 100644 --- a/internal/core/src/common/Consts.h +++ b/internal/core/src/common/Consts.h @@ -106,6 +106,7 @@ const std::string PARTITION_KEY_ISOLATION_KEY = "partition_key_isolation"; const std::string STORAGE_VERSION_KEY = "storage_version"; const std::string DIM_KEY = "dim"; const std::string DATA_TYPE_KEY = "data_type"; +const std::string ELEMENT_TYPE_KEY = "element_type"; const std::string INDEX_NUM_ROWS_KEY = "index_num_rows"; // storage version @@ -115,3 +116,6 @@ const int64_t STORAGE_V2 = 2; const std::string UNKNOW_CAST_FUNCTION_NAME = "unknown"; const int64_t DEFAULT_SHORT_COLUMN_GROUP_ID = 0; + +// VectorArray related, used for fetch metadata from Arrow schema +const std::string ELEMENT_TYPE_KEY_FOR_ARROW = "elementType"; diff --git a/internal/core/src/common/FieldData.cpp b/internal/core/src/common/FieldData.cpp index c60399f20d..9fafec723e 100644 --- a/internal/core/src/common/FieldData.cpp +++ b/internal/core/src/common/FieldData.cpp @@ -309,18 +309,70 @@ FieldDataImpl::FillFieldData( return FillFieldData(values.data(), element_count); } case DataType::VECTOR_ARRAY: { - auto array_array = - std::dynamic_pointer_cast(array); + auto list_array = + std::dynamic_pointer_cast(array); + AssertInfo(list_array != nullptr, + "Failed to cast to ListArray for VECTOR_ARRAY"); + + auto vector_array_field = + dynamic_cast*>(this); + AssertInfo(vector_array_field != nullptr, + "Failed to cast to FieldData"); + int64_t dim = vector_array_field->get_dim(); + DataType element_type = vector_array_field->get_element_type(); + + AssertInfo(dim > 0, "Invalid dimension {} in VECTOR_ARRAY", dim); + AssertInfo(element_type != DataType::NONE, + "Element type not set for VECTOR_ARRAY"); + + auto values_array = list_array->values(); std::vector values(element_count); - for (size_t index = 0; index < element_count; ++index) { - VectorFieldProto field_data; - if (array_array->GetString(index) == "") { - ThrowInfo(DataTypeInvalid, "empty vector array"); + + switch (element_type) { + case DataType::VECTOR_FLOAT: { + auto float_array = + std::dynamic_pointer_cast( + values_array); + AssertInfo( + float_array != nullptr, + "Expected FloatArray for VECTOR_FLOAT element type"); + + for (size_t index = 0; index < element_count; ++index) { + int64_t start_offset = list_array->value_offset(index); + int64_t end_offset = + list_array->value_offset(index + 1); + int64_t num_floats = end_offset - start_offset; + AssertInfo(num_floats % dim == 0, + "Invalid data: number of floats ({}) not " + "divisible by " + "dimension ({})", + num_floats, + dim); + + int num_vectors = num_floats / dim; + const float* data_ptr = + float_array->raw_values() + start_offset; + values[index] = + VectorArray(static_cast(data_ptr), + num_vectors, + dim, + element_type); + } + break; } - auto success = - field_data.ParseFromString(array_array->GetString(index)); - AssertInfo(success, "parse from string failed"); - values[index] = VectorArray(field_data); + case DataType::VECTOR_BINARY: + case DataType::VECTOR_FLOAT16: + case DataType::VECTOR_BFLOAT16: + case DataType::VECTOR_INT8: + ThrowInfo( + NotImplemented, + "Element type {} in VectorArray not implemented yet", + GetDataTypeName(element_type)); + break; + default: + ThrowInfo(DataTypeInvalid, + "Unsupported element type {} in VectorArray", + GetDataTypeName(element_type)); } return FillFieldData(values.data(), element_count); } diff --git a/internal/core/src/common/FieldData.h b/internal/core/src/common/FieldData.h index 36b45c965c..fa82c59fc4 100644 --- a/internal/core/src/common/FieldData.h +++ b/internal/core/src/common/FieldData.h @@ -83,14 +83,29 @@ class FieldData : public FieldDataArrayImpl { template <> class FieldData : public FieldDataVectorArrayImpl { public: - explicit FieldData(DataType data_type, int64_t buffered_num_rows = 0) - : FieldDataVectorArrayImpl(data_type, buffered_num_rows) { + explicit FieldData(int64_t dim, + DataType element_type, + int64_t buffered_num_rows = 0) + : FieldDataVectorArrayImpl(DataType::VECTOR_ARRAY, buffered_num_rows), + dim_(dim), + element_type_(element_type) { + AssertInfo(element_type != DataType::NONE, + "element_type must be specified for VECTOR_ARRAY"); } int64_t get_dim() const override { - ThrowInfo(Unsupported, - "Call get_dim on FieldData is not supported"); + return dim_; + } + + DataType + get_element_type() const { + return element_type_; + } + + void + set_element_type(DataType element_type) { + element_type_ = element_type; } const VectorArray* @@ -101,6 +116,10 @@ class FieldData : public FieldDataVectorArrayImpl { "subscript position don't has valid value"); return &data_[offset]; } + + private: + int64_t dim_; + DataType element_type_; }; template <> diff --git a/internal/core/src/common/FieldMeta.cpp b/internal/core/src/common/FieldMeta.cpp index 535a727ade..a745e2737d 100644 --- a/internal/core/src/common/FieldMeta.cpp +++ b/internal/core/src/common/FieldMeta.cpp @@ -85,6 +85,11 @@ FieldMeta::ParseFrom(const milvus::proto::schema::FieldSchema& schema_proto) { auto data_type = DataType(schema_proto.data_type()); auto element_type = DataType(schema_proto.element_type()); + if (data_type == DataType::VECTOR_ARRAY) { + AssertInfo(element_type != DataType::NONE, + "element_type must be specified for VECTOR_ARRAY"); + } + auto default_value = [&]() -> std::optional { if (!schema_proto.has_default_value()) { return std::nullopt; diff --git a/internal/core/src/common/Schema.cpp b/internal/core/src/common/Schema.cpp index 858830ddc9..5e0810e173 100644 --- a/internal/core/src/common/Schema.cpp +++ b/internal/core/src/common/Schema.cpp @@ -90,9 +90,19 @@ Schema::ConvertToArrowSchema() const { !IsSparseFloatVectorDataType(meta.get_data_type()) ? meta.get_dim() : 1; + + std::shared_ptr arrow_data_type = nullptr; + auto data_type = meta.get_data_type(); + if (data_type == DataType::VECTOR_ARRAY) { + arrow_data_type = + GetArrowDataTypeForVectorArray(meta.get_element_type()); + } else { + arrow_data_type = GetArrowDataType(data_type, dim); + } + auto arrow_field = std::make_shared( meta.get_name().get(), - GetArrowDataType(meta.get_data_type(), dim), + arrow_data_type, meta.is_nullable(), arrow::key_value_metadata({milvus_storage::ARROW_FIELD_ID_KEY}, {std::to_string(meta.get_id().get())})); diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h index adb2d43963..5777a2b88b 100644 --- a/internal/core/src/common/Types.h +++ b/internal/core/src/common/Types.h @@ -193,8 +193,6 @@ GetArrowDataType(DataType data_type, int dim = 1) { return arrow::binary(); case DataType::VECTOR_INT8: return arrow::fixed_size_binary(dim); - case DataType::VECTOR_ARRAY: - return arrow::binary(); default: { ThrowInfo(DataTypeInvalid, fmt::format("failed to get data type, invalid type {}", @@ -203,6 +201,20 @@ GetArrowDataType(DataType data_type, int dim = 1) { } } +inline std::shared_ptr +GetArrowDataTypeForVectorArray(DataType elem_type) { + switch (elem_type) { + case DataType::VECTOR_FLOAT: + return arrow::list(arrow::float32()); + default: { + ThrowInfo(DataTypeInvalid, + fmt::format("failed to get arrow type for vector array, " + "invalid type {}", + elem_type)); + } + } +} + template inline size_t GetVecRowSize(int64_t dim) { @@ -454,10 +466,12 @@ using FieldName = fluent::NamedType; -// field id -> (field name, field type, binlog paths) -using OptFieldT = std::unordered_map< - int64_t, - std::tuple>>; +// field id -> (field name, field type, element type, binlog paths) +using OptFieldT = std::unordered_map>>; using SegmentInsertFiles = std::vector>; diff --git a/internal/core/src/common/VectorArray.h b/internal/core/src/common/VectorArray.h index c855d90780..936ca7a417 100644 --- a/internal/core/src/common/VectorArray.h +++ b/internal/core/src/common/VectorArray.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include "FieldMeta.h" #include "Types.h" @@ -31,6 +32,30 @@ class VectorArray : public milvus::VectorTrait { ~VectorArray() = default; + VectorArray(const void* data, + int num_vectors, + int64_t dim, + DataType element_type) + : dim_(dim), length_(num_vectors), element_type_(element_type) { + assert(data != nullptr); + assert(num_vectors > 0); + assert(dim > 0); + + switch (element_type) { + case DataType::VECTOR_FLOAT: + size_ = num_vectors * dim * sizeof(float); + break; + default: + ThrowInfo(NotImplemented, + "Direct VectorArray construction only supports " + "VECTOR_FLOAT, got {}", + GetDataTypeName(element_type)); + } + + data_ = std::make_unique(size_); + std::memcpy(data_.get(), data, size_); + } + // One row of VectorFieldProto explicit VectorArray(const VectorFieldProto& vector_field) { dim_ = vector_field.dim(); diff --git a/internal/core/src/index/Utils.cpp b/internal/core/src/index/Utils.cpp index 9981e55795..4b5d37c26b 100644 --- a/internal/core/src/index/Utils.cpp +++ b/internal/core/src/index/Utils.cpp @@ -356,8 +356,8 @@ AssembleIndexDatas(std::map& index_datas, int slice_num = item[SLICE_NUM]; auto total_len = static_cast(item[TOTAL_LEN]); // build index skip null value, so not need to set nullable == true - auto new_field_data = - storage::CreateFieldData(DataType::INT8, false, 1, total_len); + auto new_field_data = storage::CreateFieldData( + DataType::INT8, DataType::NONE, false, 1, total_len); for (auto i = 0; i < slice_num; ++i) { std::string file_name = GenSlicedFileName(prefix, i); diff --git a/internal/core/src/index/json_stats/JsonKeyStats.cpp b/internal/core/src/index/json_stats/JsonKeyStats.cpp index 2bb74d28d6..cf71fd4504 100644 --- a/internal/core/src/index/json_stats/JsonKeyStats.cpp +++ b/internal/core/src/index/json_stats/JsonKeyStats.cpp @@ -608,6 +608,7 @@ JsonKeyStats::Build(const Config& config) { }(); auto field_data = storage::CreateFieldData( static_cast(field_schema.data_type()), + DataType::NONE, true, 1, lack_binlog_rows.value()); diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index f0a6fc30fe..46bce33b93 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -125,10 +125,11 @@ get_opt_field(const ::google::protobuf::RepeatedPtrField< opt_fields_map[field_id] = { field_info.field_name(), static_cast(field_info.field_type()), + static_cast(field_info.element_type()), {}}; } for (const auto& str : field_info.data_paths()) { - std::get<2>(opt_fields_map[field_id]).emplace_back(str); + std::get<3>(opt_fields_map[field_id]).emplace_back(str); } } @@ -179,6 +180,7 @@ get_config(std::unique_ptr& info) { } config[DIM_KEY] = info->dim(); config[DATA_TYPE_KEY] = info->field_schema().data_type(); + config[ELEMENT_TYPE_KEY] = info->field_schema().element_type(); return config; } diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index 88fef5b040..9e6c74b2cd 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -294,11 +294,12 @@ SegmentGrowingImpl::load_field_data_internal(const LoadFieldDataInfo& infos) { AssertInfo(field_meta.is_nullable(), "nullable must be true when lack rows"); auto lack_num = info.row_count - total; - auto field_data = storage::CreateFieldData( - static_cast(field_meta.get_data_type()), - true, - 1, - lack_num); + auto field_data = + storage::CreateFieldData(field_meta.get_data_type(), + field_meta.get_element_type(), + true, + 1, + lack_num); field_data->FillFieldData(field_meta.default_value(), lack_num); channel->push(field_data); } @@ -494,6 +495,7 @@ SegmentGrowingImpl::load_column_group_data_internal( auto data_type = field.second.get_data_type(); auto field_data = storage::CreateFieldData( data_type, + field.second.get_element_type(), field.second.is_nullable(), IsVectorDataType(data_type) && !IsSparseFloatVectorDataType(data_type) diff --git a/internal/core/src/storage/DataCodec.cpp b/internal/core/src/storage/DataCodec.cpp index e02eddb5d1..06c96e9709 100644 --- a/internal/core/src/storage/DataCodec.cpp +++ b/internal/core/src/storage/DataCodec.cpp @@ -129,7 +129,8 @@ DeserializeFileData(const std::shared_ptr input_data, AssertInfo( field_data->get_num_rows() == 1, "wrong length of string num in old index binlog file"); - auto new_field_data = CreateFieldData(DataType::INT8, nullable); + auto new_field_data = + CreateFieldData(DataType::INT8, DataType::NONE, nullable); new_field_data->FillFieldData( (*static_cast(field_data->RawValue(0))) .c_str(), diff --git a/internal/core/src/storage/DiskFileManagerImpl.cpp b/internal/core/src/storage/DiskFileManagerImpl.cpp index 4f1d2bfac2..760875d610 100644 --- a/internal/core/src/storage/DiskFileManagerImpl.cpp +++ b/internal/core/src/storage/DiskFileManagerImpl.cpp @@ -534,7 +534,11 @@ template std::string DiskFileManagerImpl::cache_raw_data_to_disk_storage_v2(const Config& config) { auto data_type = index::GetValueFromConfig(config, DATA_TYPE_KEY); + auto element_type = + index::GetValueFromConfig(config, ELEMENT_TYPE_KEY); AssertInfo(data_type.has_value(), "data type is empty when build index"); + AssertInfo(element_type.has_value(), + "element type is empty when build index"); auto dim = index::GetValueFromConfig(config, DIM_KEY).value_or(0); auto segment_insert_files = index::GetValueFromConfig>>( @@ -560,6 +564,7 @@ DiskFileManagerImpl::cache_raw_data_to_disk_storage_v2(const Config& config) { auto field_datas = GetFieldDatasFromStorageV2(all_remote_files, GetFieldDataMeta().field_id, data_type.value(), + element_type.value(), dim, fs_); for (auto& field_data : field_datas) { @@ -776,14 +781,19 @@ DiskFileManagerImpl::CacheOptFieldToDisk(const Config& config) { std::unordered_set actual_field_ids; for (auto& [field_id, tup] : fields_map) { const auto& field_type = std::get<1>(tup); + const auto& element_type = std::get<2>(tup); std::vector field_datas; // fetch scalar data from storage v2 if (storage_version == STORAGE_V2) { - field_datas = GetFieldDatasFromStorageV2( - remote_files_storage_v2, field_id, field_type, 1, fs_); + field_datas = GetFieldDatasFromStorageV2(remote_files_storage_v2, + field_id, + field_type, + element_type, + 1, + fs_); } else { // original way - auto& field_paths = std::get<2>(tup); + auto& field_paths = std::get<3>(tup); if (0 == field_paths.size()) { LOG_WARN("optional field {} has no data", field_id); return ""; diff --git a/internal/core/src/storage/Event.cpp b/internal/core/src/storage/Event.cpp index 75c0a7634d..7083d3ace0 100644 --- a/internal/core/src/storage/Event.cpp +++ b/internal/core/src/storage/Event.cpp @@ -257,10 +257,17 @@ BaseEventData::Serialize() { auto field_data = payload_reader->get_field_data(); auto data_type = field_data->get_data_type(); std::shared_ptr payload_writer; - if (IsVectorDataType(data_type) && - // each element will be serialized as bytes so no need dim info - data_type != DataType::VECTOR_ARRAY && - !IsSparseFloatVectorDataType(data_type)) { + + if (data_type == DataType::VECTOR_ARRAY) { + auto vector_array_field = + std::dynamic_pointer_cast>(field_data); + AssertInfo(vector_array_field != nullptr, + "Failed to cast to FieldData"); + auto element_type = vector_array_field->get_element_type(); + payload_writer = std::make_unique( + data_type, field_data->get_dim(), element_type); + } else if (IsVectorDataType(data_type) && + !IsSparseFloatVectorDataType(data_type)) { payload_writer = std::make_unique( data_type, field_data->get_dim(), field_data->IsNullable()); } else { @@ -324,19 +331,14 @@ BaseEventData::Serialize() { break; } case DataType::VECTOR_ARRAY: { - for (size_t offset = 0; offset < field_data->get_num_rows(); - ++offset) { - auto array = static_cast( - field_data->RawValue(offset)); - auto array_string = - array->output_data().SerializeAsString(); - auto size = array_string.size(); - - // todo(SapdeA): it maybe better to serialize vectors one by one - payload_writer->add_one_binary_payload( - reinterpret_cast(array_string.c_str()), - size); - } + auto payload = + Payload{data_type, + static_cast(field_data->Data()), + field_data->ValidData(), + field_data->get_num_rows(), + field_data->get_dim(), + field_data->IsNullable()}; + payload_writer->add_payload(payload); break; } default: { diff --git a/internal/core/src/storage/MemFileManagerImpl.cpp b/internal/core/src/storage/MemFileManagerImpl.cpp index f8dc1281fb..1f4080a624 100644 --- a/internal/core/src/storage/MemFileManagerImpl.cpp +++ b/internal/core/src/storage/MemFileManagerImpl.cpp @@ -205,6 +205,10 @@ MemFileManagerImpl::cache_raw_data_to_memory_storage_v2(const Config& config) { auto data_type = index::GetValueFromConfig(config, DATA_TYPE_KEY); AssertInfo(data_type.has_value(), "[StorageV2] data type is empty when build index"); + auto element_type = + index::GetValueFromConfig(config, ELEMENT_TYPE_KEY); + AssertInfo(element_type.has_value(), + "[StorageV2] element type is empty when build index"); auto dim = index::GetValueFromConfig(config, DIM_KEY).value_or(0); auto segment_insert_files = index::GetValueFromConfig>>( @@ -216,8 +220,12 @@ MemFileManagerImpl::cache_raw_data_to_memory_storage_v2(const Config& config) { for (auto& files : remote_files) { SortByPath(files); } - auto field_datas = GetFieldDatasFromStorageV2( - remote_files, field_meta_.field_id, data_type.value(), dim, fs_); + auto field_datas = GetFieldDatasFromStorageV2(remote_files, + field_meta_.field_id, + data_type.value(), + element_type.value(), + dim, + fs_); // field data list could differ for storage v2 group list return field_datas; } @@ -309,7 +317,7 @@ MemFileManagerImpl::cache_opt_field_memory(const Config& config) { for (auto& [field_id, tup] : fields_map) { const auto& field_type = std::get<1>(tup); - auto& field_paths = std::get<2>(tup); + auto& field_paths = std::get<3>(tup); if (0 == field_paths.size()) { LOG_WARN("optional field {} has no data", field_id); return {}; @@ -353,9 +361,10 @@ MemFileManagerImpl::cache_opt_field_memory_v2(const Config& config) { std::unordered_map>> res; for (auto& [field_id, tup] : fields_map) { const auto& field_type = std::get<1>(tup); + const auto& element_type = std::get<2>(tup); auto field_datas = GetFieldDatasFromStorageV2( - remote_files, field_id, field_type, 1, fs_); + remote_files, field_id, field_type, element_type, 1, fs_); res[field_id] = GetOptFieldIvfData(field_type, field_datas); } diff --git a/internal/core/src/storage/PayloadReader.cpp b/internal/core/src/storage/PayloadReader.cpp index 64dcfa5109..bd10f825f8 100644 --- a/internal/core/src/storage/PayloadReader.cpp +++ b/internal/core/src/storage/PayloadReader.cpp @@ -73,19 +73,58 @@ PayloadReader::init(const uint8_t* data, int length, bool is_field_data) { // dim is unused for sparse float vector dim_ = (IsVectorDataType(column_type_) && - !IsSparseFloatVectorDataType(column_type_)) && - !IsVectorArrayDataType(column_type_) + !IsVectorArrayDataType(column_type_) && + !IsSparseFloatVectorDataType(column_type_)) ? GetDimensionFromFileMetaData( file_meta->schema()->Column(column_index), column_type_) : 1; + + // For VectorArray, get element type and dim from Arrow schema metadata + auto element_type = DataType::NONE; + if (IsVectorArrayDataType(column_type_)) { + std::shared_ptr arrow_schema; + st = arrow_reader->GetSchema(&arrow_schema); + AssertInfo(st.ok(), "Failed to get arrow schema for VectorArray"); + AssertInfo(arrow_schema->num_fields() == 1, + "VectorArray should have exactly 1 field, got {}", + arrow_schema->num_fields()); + + auto field = arrow_schema->field(0); + AssertInfo(field->HasMetadata(), + "VectorArray field is missing metadata"); + + auto metadata = field->metadata(); + AssertInfo(metadata != nullptr, "VectorArray metadata is null"); + + // Get element type + AssertInfo( + metadata->Contains(ELEMENT_TYPE_KEY_FOR_ARROW), + "VectorArray metadata missing required 'elementType' field"); + auto element_type_str = + metadata->Get(ELEMENT_TYPE_KEY_FOR_ARROW).ValueOrDie(); + auto element_type_int = std::stoi(element_type_str); + element_type = static_cast(element_type_int); + + // Get dimension from metadata + AssertInfo(metadata->Contains(DIM_KEY), + "VectorArray metadata missing required 'dim' field"); + auto dim_str = metadata->Get(DIM_KEY).ValueOrDie(); + dim_ = std::stoi(dim_str); + AssertInfo( + dim_ > 0, "VectorArray dim must be positive, got {}", dim_); + } + std::shared_ptr<::arrow::RecordBatchReader> rb_reader; st = arrow_reader->GetRecordBatchReader(&rb_reader); AssertInfo(st.ok(), "get record batch reader"); if (is_field_data) { auto total_num_rows = file_meta->num_rows(); - field_data_ = - CreateFieldData(column_type_, nullable_, dim_, total_num_rows); + + // Create FieldData, passing element_type for VectorArray + field_data_ = CreateFieldData( + column_type_, element_type, nullable_, dim_, total_num_rows); + for (arrow::Result> maybe_batch : *rb_reader) { AssertInfo(maybe_batch.ok(), "get batch record success"); diff --git a/internal/core/src/storage/PayloadWriter.cpp b/internal/core/src/storage/PayloadWriter.cpp index 68e16f719c..e6ce1a0cf3 100644 --- a/internal/core/src/storage/PayloadWriter.cpp +++ b/internal/core/src/storage/PayloadWriter.cpp @@ -39,6 +39,21 @@ PayloadWriter::PayloadWriter(const DataType column_type, int dim, bool nullable) init_dimension(dim); } +// create payload writer for VectorArray with element_type +PayloadWriter::PayloadWriter(const DataType column_type, + int dim, + DataType element_type) + // VECTOR_ARRAY is not nullable + : column_type_(column_type), nullable_(false), element_type_(element_type) { + AssertInfo(column_type == DataType::VECTOR_ARRAY, + "This constructor is only for VECTOR_ARRAY"); + AssertInfo(element_type != DataType::NONE, + "element_type must be specified for VECTOR_ARRAY"); + dimension_ = dim; + builder_ = CreateArrowBuilder(column_type, element_type, dim); + schema_ = CreateArrowSchema(column_type, dim, element_type); +} + void PayloadWriter::init_dimension(int dim) { if (dimension_.has_value()) { @@ -48,7 +63,7 @@ PayloadWriter::init_dimension(int dim) { } dimension_ = dim; - builder_ = CreateArrowBuilder(column_type_, dim); + builder_ = CreateArrowBuilder(column_type_, element_type_, dim); schema_ = CreateArrowSchema(column_type_, dim, nullable_); } @@ -94,6 +109,16 @@ PayloadWriter::finish() { auto table = arrow::Table::Make(schema_, {array}); output_ = std::make_shared(); auto mem_pool = arrow::default_memory_pool(); + + std::shared_ptr arrow_properties = + parquet::default_arrow_writer_properties(); + if (column_type_ == DataType::VECTOR_ARRAY) { + // For VectorArray, we need to store schema metadata + parquet::ArrowWriterProperties::Builder arrow_props_builder; + arrow_props_builder.store_schema(); + arrow_properties = arrow_props_builder.build(); + } + ast = parquet::arrow::WriteTable(*table, mem_pool, output_, @@ -101,7 +126,8 @@ PayloadWriter::finish() { parquet::WriterProperties::Builder() .compression(arrow::Compression::ZSTD) ->compression_level(3) - ->build()); + ->build(), + arrow_properties); AssertInfo(ast.ok(), ast.ToString()); } diff --git a/internal/core/src/storage/PayloadWriter.h b/internal/core/src/storage/PayloadWriter.h index 86ca281bb6..7f60e86021 100644 --- a/internal/core/src/storage/PayloadWriter.h +++ b/internal/core/src/storage/PayloadWriter.h @@ -27,6 +27,10 @@ class PayloadWriter { public: explicit PayloadWriter(const DataType column_type, int dim, bool nullable); explicit PayloadWriter(const DataType column_type, bool nullable); + // Constructor for VectorArray with element_type + explicit PayloadWriter(const DataType column_type, + int dim, + DataType element_type); ~PayloadWriter() = default; void @@ -59,6 +63,7 @@ class PayloadWriter { private: DataType column_type_; bool nullable_; + DataType element_type_ = DataType::NONE; // For VectorArray std::shared_ptr builder_; std::shared_ptr schema_; std::shared_ptr output_; diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index f94db23c6d..502d345219 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -18,6 +18,7 @@ #include "arrow/array/builder_binary.h" #include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" #include "arrow/scalar.h" #include "arrow/type_fwd.h" #include "common/type_c.h" @@ -48,6 +49,8 @@ #include "storage/Types.h" #include "storage/Util.h" #include "common/Common.h" +#include "common/Types.h" +#include "common/VectorArray.h" #include "storage/ThreadPools.h" #include "storage/MemFileManagerImpl.h" #include "storage/DiskFileManagerImpl.h" @@ -226,6 +229,80 @@ AddPayloadToArrowBuilder(std::shared_ptr builder, "add_one_binary_payload", data_type); } + case DataType::VECTOR_ARRAY: { + auto list_builder = + std::dynamic_pointer_cast(builder); + AssertInfo(list_builder != nullptr, + "builder must be ListBuilder for VECTOR_ARRAY"); + + auto vector_arrays = reinterpret_cast(raw_data); + + if (length > 0) { + auto element_type = vector_arrays[0].get_element_type(); + + switch (element_type) { + case DataType::VECTOR_FLOAT: { + auto value_builder = static_cast( + list_builder->value_builder()); + AssertInfo(value_builder != nullptr, + "value_builder must be FloatBuilder for " + "FloatVector"); + + arrow::Status ast; + for (int i = 0; i < length; ++i) { + auto status = list_builder->Append(); + AssertInfo(status.ok(), + "Failed to append list: {}", + status.ToString()); + + const auto& array = vector_arrays[i]; + AssertInfo( + array.get_element_type() == + DataType::VECTOR_FLOAT, + "Inconsistent element types in VectorArray"); + + int num_vectors = array.length(); + int dim = array.dim(); + + for (int j = 0; j < num_vectors; ++j) { + auto vec_data = array.get_data(j); + ast = + value_builder->AppendValues(vec_data, dim); + AssertInfo(ast.ok(), + "Failed to append list: {}", + ast.ToString()); + } + } + break; + } + case DataType::VECTOR_BINARY: + ThrowInfo( + NotImplemented, + "BinaryVector in VectorArray not implemented yet"); + break; + case DataType::VECTOR_FLOAT16: + ThrowInfo( + NotImplemented, + "Float16Vector in VectorArray not implemented yet"); + break; + case DataType::VECTOR_BFLOAT16: + ThrowInfo(NotImplemented, + "BFloat16Vector in VectorArray not " + "implemented yet"); + break; + case DataType::VECTOR_INT8: + ThrowInfo( + NotImplemented, + "Int8Vector in VectorArray not implemented yet"); + break; + default: + ThrowInfo(DataTypeInvalid, + "Unsupported element type in VectorArray: {}", + element_type); + } + } + break; + } default: { ThrowInfo(DataTypeInvalid, "unsupported data type {}", data_type); } @@ -299,7 +376,6 @@ CreateArrowBuilder(DataType data_type) { return std::make_shared(); } case DataType::ARRAY: - case DataType::VECTOR_ARRAY: case DataType::JSON: { return std::make_shared(); } @@ -315,7 +391,7 @@ CreateArrowBuilder(DataType data_type) { } std::shared_ptr -CreateArrowBuilder(DataType data_type, int dim) { +CreateArrowBuilder(DataType data_type, DataType element_type, int dim) { switch (static_cast(data_type)) { case DataType::VECTOR_FLOAT: { AssertInfo(dim > 0, "invalid dim value: {}", dim); @@ -342,6 +418,27 @@ CreateArrowBuilder(DataType data_type, int dim) { return std::make_shared( arrow::fixed_size_binary(dim * sizeof(int8))); } + case DataType::VECTOR_ARRAY: { + AssertInfo(dim > 0, "invalid dim value"); + AssertInfo(element_type != DataType::NONE, + "element_type must be specified for VECTOR_ARRAY"); + + std::shared_ptr value_builder; + switch (element_type) { + case DataType::VECTOR_FLOAT: { + value_builder = std::make_shared(); + break; + } + default: { + ThrowInfo(DataTypeInvalid, + "unsupported element type {} for VECTOR_ARRAY", + GetDataTypeName(element_type)); + } + } + + return std::make_shared( + arrow::default_memory_pool(), value_builder); + } default: { ThrowInfo( DataTypeInvalid, "unsupported vector data type {}", data_type); @@ -434,7 +531,6 @@ CreateArrowSchema(DataType data_type, bool nullable) { {arrow::field("val", arrow::utf8(), nullable)}); } case DataType::ARRAY: - case DataType::VECTOR_ARRAY: case DataType::JSON: { return arrow::schema( {arrow::field("val", arrow::binary(), nullable)}); @@ -491,6 +587,13 @@ CreateArrowSchema(DataType data_type, int dim, bool nullable) { arrow::fixed_size_binary(dim * sizeof(int8)), nullable)}); } + case DataType::VECTOR_ARRAY: { + // VectorArray should not use this overload - should call the one with element_type + ThrowInfo( + NotImplemented, + "VectorArray requires element_type parameter. Use " + "CreateArrowSchema(data_type, dim, element_type, nullable)"); + } default: { ThrowInfo( DataTypeInvalid, "unsupported vector data type {}", data_type); @@ -498,6 +601,22 @@ CreateArrowSchema(DataType data_type, int dim, bool nullable) { } } +std::shared_ptr +CreateArrowSchema(DataType data_type, int dim, DataType element_type) { + AssertInfo(data_type == DataType::VECTOR_ARRAY, + "This overload is only for VECTOR_ARRAY type"); + AssertInfo(dim > 0, "invalid dim value"); + + auto value_type = GetArrowDataTypeForVectorArray(element_type); + auto metadata = arrow::KeyValueMetadata::Make( + {ELEMENT_TYPE_KEY_FOR_ARROW, DIM_KEY}, + {std::to_string(static_cast(element_type)), std::to_string(dim)}); + + // VECTOR_ARRAY is not nullable + auto field = arrow::field("val", value_type, false)->WithMetadata(metadata); + return arrow::schema({field}); +} + int GetDimensionFromFileMetaData(const parquet::ColumnDescriptor* schema, DataType data_type) { @@ -527,60 +646,6 @@ GetDimensionFromFileMetaData(const parquet::ColumnDescriptor* schema, } } -int -GetDimensionFromArrowArray(std::shared_ptr data, - DataType data_type) { - switch (data_type) { - case DataType::VECTOR_FLOAT: { - AssertInfo( - data->type()->id() == arrow::Type::type::FIXED_SIZE_BINARY, - "inconsistent data type: {}", - data->type_id()); - auto array = - std::dynamic_pointer_cast(data); - return array->byte_width() / sizeof(float); - } - case DataType::VECTOR_BINARY: { - AssertInfo( - data->type()->id() == arrow::Type::type::FIXED_SIZE_BINARY, - "inconsistent data type: {}", - data->type_id()); - auto array = - std::dynamic_pointer_cast(data); - return array->byte_width() * 8; - } - case DataType::VECTOR_FLOAT16: { - AssertInfo( - data->type()->id() == arrow::Type::type::FIXED_SIZE_BINARY, - "inconsistent data type: {}", - data->type_id()); - auto array = - std::dynamic_pointer_cast(data); - return array->byte_width() / sizeof(float16); - } - case DataType::VECTOR_BFLOAT16: { - AssertInfo( - data->type()->id() == arrow::Type::type::FIXED_SIZE_BINARY, - "inconsistent data type: {}", - data->type_id()); - auto array = - std::dynamic_pointer_cast(data); - return array->byte_width() / sizeof(bfloat16); - } - case DataType::VECTOR_INT8: { - AssertInfo( - data->type()->id() == arrow::Type::type::FIXED_SIZE_BINARY, - "inconsistent data type: {}", - data->type_id()); - auto array = - std::dynamic_pointer_cast(data); - return array->byte_width() / sizeof(int8); - } - default: - ThrowInfo(DataTypeInvalid, "unsupported data type {}", data_type); - } -} - std::string GenIndexPathIdentifier(int64_t build_id, int64_t index_version, @@ -746,7 +811,8 @@ EncodeAndUploadIndexSlice(ChunkManager* chunk_manager, index_data = std::make_shared(buf, batch_size); // index-build tasks assigned from new milvus-coord nodes to none-encoding } else { - auto field_data = CreateFieldData(DataType::INT8, false); + auto field_data = + CreateFieldData(DataType::INT8, DataType::NONE, false); field_data->FillFieldData(buf, batch_size); auto payload_reader = std::make_shared(field_data); index_data = std::make_shared(payload_reader); @@ -967,6 +1033,7 @@ InitArrowFileSystem(milvus::storage::StorageConfig storage_config) { FieldDataPtr CreateFieldData(const DataType& type, + const DataType& element_type, bool nullable, int64_t dim, int64_t total_num_rows) { @@ -1025,8 +1092,8 @@ CreateFieldData(const DataType& type, return std::make_shared>( dim, type, total_num_rows); case DataType::VECTOR_ARRAY: - return std::make_shared>(type, - total_num_rows); + return std::make_shared>( + dim, element_type, total_num_rows); default: ThrowInfo(DataTypeInvalid, "CreateFieldData not support data type " + @@ -1068,7 +1135,16 @@ MergeFieldData(std::vector& data_array) { for (const auto& data : data_array) { total_length += data->Length(); } + + auto element_type = DataType::NONE; + auto vector_array_data = + dynamic_cast*>(data_array[0].get()); + if (vector_array_data) { + element_type = vector_array_data->get_element_type(); + } + auto merged_data = storage::CreateFieldData(data_array[0]->get_data_type(), + element_type, data_array[0]->IsNullable()); merged_data->Reserve(total_length); for (const auto& data : data_array) { @@ -1114,6 +1190,7 @@ std::vector GetFieldDatasFromStorageV2(std::vector>& remote_files, int64_t field_id, DataType data_type, + DataType element_type, int64_t dim, milvus_storage::ArrowFileSystemPtr fs) { AssertInfo(remote_files.size() > 0, "[StorageV2] remote files size is 0"); @@ -1218,8 +1295,11 @@ GetFieldDatasFromStorageV2(std::vector>& remote_files, num_rows += table_info.table->num_rows(); chunked_arrays.push_back(table_info.table->column(col_offset)); } - auto field_data = storage::CreateFieldData( - data_type, field_schema->nullable(), dim, num_rows); + auto field_data = storage::CreateFieldData(data_type, + element_type, + field_schema->nullable(), + dim, + num_rows); for (const auto& chunked_array : chunked_arrays) { field_data->FillFieldData(chunked_array); } @@ -1259,6 +1339,7 @@ CacheRawDataAndFillMissing(const MemFileManagerImplPtr& file_manager, }(); auto field_data = storage::CreateFieldData( static_cast(field_schema.data_type()), + static_cast(field_schema.element_type()), true, 1, lack_binlog_rows); diff --git a/internal/core/src/storage/Util.h b/internal/core/src/storage/Util.h index 611d967070..326605506a 100644 --- a/internal/core/src/storage/Util.h +++ b/internal/core/src/storage/Util.h @@ -59,7 +59,7 @@ std::shared_ptr CreateArrowBuilder(DataType data_type); std::shared_ptr -CreateArrowBuilder(DataType data_type, int dim); +CreateArrowBuilder(DataType data_type, DataType element_type, int dim); /// \brief Utility function to create arrow:Scalar from FieldMeta.default_value /// @@ -80,14 +80,13 @@ CreateArrowSchema(DataType data_type, bool nullable); std::shared_ptr CreateArrowSchema(DataType data_type, int dim, bool nullable); +std::shared_ptr +CreateArrowSchema(DataType data_type, int dim, DataType element_type); + int GetDimensionFromFileMetaData(const parquet::ColumnDescriptor* schema, DataType data_type); -int -GetDimensionFromArrowArray(std::shared_ptr array, - DataType data_type); - std::string GenIndexPathIdentifier(int64_t build_id, int64_t index_version, @@ -186,6 +185,7 @@ std::vector GetFieldDatasFromStorageV2(std::vector>& remote_files, int64_t field_id, DataType data_type, + DataType element_type, int64_t dim, milvus_storage::ArrowFileSystemPtr fs); @@ -215,6 +215,7 @@ InitArrowFileSystem(milvus::storage::StorageConfig storage_config); FieldDataPtr CreateFieldData(const DataType& type, + const DataType& element_type, bool nullable = false, int64_t dim = 1, int64_t total_num_rows = 0); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 7982ade2ac..2001a3f387 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -112,6 +112,7 @@ set(MILVUS_TEST_FILES test_stlsort_index.cpp test_vector_array_storage_v2.cpp test_rescore.cpp + test_vector_array_chunk.cpp ) if ( INDEX_ENGINE STREQUAL "cardinal" ) diff --git a/internal/core/unittest/test_array_bitmap_index.cpp b/internal/core/unittest/test_array_bitmap_index.cpp index 51e35b7416..e03a0e4f00 100644 --- a/internal/core/unittest/test_array_bitmap_index.cpp +++ b/internal/core/unittest/test_array_bitmap_index.cpp @@ -186,7 +186,8 @@ class ArrayBitmapIndexTest : public testing::Test { segment_id, field_id, index_build_id, index_version}; data_ = GenerateArrayData(element_type, cardinality_, nb_, 10); - auto field_data = storage::CreateFieldData(DataType::ARRAY, nullable_); + auto field_data = storage::CreateFieldData( + DataType::ARRAY, DataType::NONE, nullable_); if (nullable_) { valid_data_.reserve(nb_); uint8_t* ptr = new uint8_t[(nb_ + 7) / 8]; diff --git a/internal/core/unittest/test_binlog_index.cpp b/internal/core/unittest/test_binlog_index.cpp index c331de5385..77b0791e18 100644 --- a/internal/core/unittest/test_binlog_index.cpp +++ b/internal/core/unittest/test_binlog_index.cpp @@ -76,7 +76,8 @@ class BinlogIndexTest : public ::testing::TestWithParam { schema->AddDebugField("fakevec", data_type, data_d, metric_type); auto i64_fid = schema->AddDebugField("counter", DataType::INT64); schema->set_primary_field_id(i64_fid); - vec_field_data = storage::CreateFieldData(data_type, false, data_d); + vec_field_data = + storage::CreateFieldData(data_type, DataType::NONE, false, data_d); if (data_type == DataType::VECTOR_FLOAT) { auto vec_data = GenRandomFloatVecData(data_n, data_d); diff --git a/internal/core/unittest/test_bitmap_index.cpp b/internal/core/unittest/test_bitmap_index.cpp index 5c488c961f..be8d9068b5 100644 --- a/internal/core/unittest/test_bitmap_index.cpp +++ b/internal/core/unittest/test_bitmap_index.cpp @@ -117,7 +117,8 @@ class BitmapIndexTest : public testing::Test { data_.push_back(x); } - auto field_data = storage::CreateFieldData(type_, nullable_); + auto field_data = + storage::CreateFieldData(type_, DataType::NONE, nullable_); if (nullable_) { valid_data_.reserve(nb_); uint8_t* ptr = new uint8_t[(nb_ + 7) / 8]; diff --git a/internal/core/unittest/test_chunk.cpp b/internal/core/unittest/test_chunk.cpp index 72f15c6c15..5e234e328a 100644 --- a/internal/core/unittest/test_chunk.cpp +++ b/internal/core/unittest/test_chunk.cpp @@ -37,8 +37,8 @@ using namespace milvus; TEST(chunk, test_int64_field) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT64); + auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT64, + DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; auto payload_reader = @@ -80,7 +80,7 @@ TEST(chunk, test_timestmamptz_field) { FixedVector data = { 1, 2, 3, 4, 5}; // Timestamptz is stored as int64 auto field_data = - milvus::storage::CreateFieldData(storage::DataType::TIMESTAMPTZ); + milvus::storage::CreateFieldData(DataType::TIMESTAMPTZ, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; auto payload_reader = @@ -121,8 +121,8 @@ TEST(chunk, test_timestmamptz_field) { TEST(chunk, test_variable_field) { FixedVector data = { "test1", "test2", "test3", "test4", "test5"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; @@ -164,8 +164,8 @@ TEST(chunk, test_variable_field_nullable) { "test1", "test2", "test3", "test4", "test5"}; FixedVector validity = {true, false, true, false, true}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0x15}; // 10101 in binary field_data->FillFieldData(data.data(), valid_data, data.size(), 0); delete[] valid_data; @@ -216,7 +216,8 @@ TEST(chunk, test_json_field) { auto json = Json(json_str.data(), json_str.size()); data.push_back(std::move(json)); } - auto field_data = milvus::storage::CreateFieldData(storage::DataType::JSON); + auto field_data = milvus::storage::CreateFieldData(storage::DataType::JSON, + DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; @@ -327,8 +328,8 @@ TEST(chunk, test_json_field) { TEST(chunk, test_null_int64) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT64, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT64, DataType::NONE, true); // Set up validity bitmap: 10011 (1st, 4th, and 5th are valid) uint8_t* valid_data = new uint8_t[1]{0x13}; // 10011 in binary @@ -392,8 +393,8 @@ TEST(chunk, test_array) { field_string_data.mutable_string_data()->add_data("test_array5"); auto string_array = Array(field_string_data); FixedVector data = {string_array}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::ARRAY); + auto field_data = milvus::storage::CreateFieldData(storage::DataType::ARRAY, + DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; auto payload_reader = @@ -450,8 +451,8 @@ TEST(chunk, test_null_array) { data.emplace_back(string_array); } - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::ARRAY, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::ARRAY, DataType::NONE, true); // Set up validity bitmap: 10101 (1st, 3rd, and 5th are valid) uint8_t* valid_data = new uint8_t[1]{0x15}; // 10101 in binary @@ -529,8 +530,8 @@ TEST(chunk, test_array_views) { data.emplace_back(string_array); } - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::ARRAY); + auto field_data = milvus::storage::CreateFieldData(storage::DataType::ARRAY, + DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; auto payload_reader = @@ -609,11 +610,12 @@ TEST(chunk, test_sparse_float) { auto n_rows = 100; auto vecs = milvus::segcore::GenerateRandomSparseFloatVector( n_rows, kTestSparseDim, kTestSparseVectorDensity); - auto field_data = milvus::storage::CreateFieldData( - storage::DataType::VECTOR_SPARSE_U32_F32, - false, - kTestSparseDim, - n_rows); + auto field_data = + milvus::storage::CreateFieldData(DataType::VECTOR_SPARSE_U32_F32, + DataType::NONE, + false, + kTestSparseDim, + n_rows); field_data->FillFieldData(vecs.get(), n_rows); storage::InsertEventData event_data; @@ -670,8 +672,8 @@ TEST(chunk, test_lower_bound_string) { "kiwi", "lemon"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; @@ -746,8 +748,8 @@ TEST(chunk, test_upper_bound_string) { "kiwi", "lemon"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; @@ -815,8 +817,8 @@ TEST(chunk, test_binary_search_methods_edge_cases) { // Test with single element FixedVector single_data = {"middle"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(single_data.data(), single_data.size()); storage::InsertEventData event_data; @@ -862,8 +864,8 @@ TEST(chunk, test_binary_search_methods_duplicates) { FixedVector data = { "apple", "apple", "banana", "banana", "banana", "cherry"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; @@ -912,8 +914,8 @@ TEST(chunk, test_binary_search_methods_comparison) { FixedVector data = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; diff --git a/internal/core/unittest/test_chunked_column_group.cpp b/internal/core/unittest/test_chunked_column_group.cpp index 6d6f4079a7..b30527ff17 100644 --- a/internal/core/unittest/test_chunked_column_group.cpp +++ b/internal/core/unittest/test_chunked_column_group.cpp @@ -39,8 +39,8 @@ using namespace milvus::storage; std::shared_ptr create_chunk(const FixedVector& data) { - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT64); + auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT64, + DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; auto payload_reader = @@ -74,8 +74,8 @@ create_chunk(const FixedVector& data) { // Helper function to create chunks for string data std::shared_ptr create_chunk(const FixedVector& data) { - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE); field_data->FillFieldData(data.data(), data.size()); storage::InsertEventData event_data; diff --git a/internal/core/unittest/test_data_codec.cpp b/internal/core/unittest/test_data_codec.cpp index da541618fe..3351c7e684 100644 --- a/internal/core/unittest/test_data_codec.cpp +++ b/internal/core/unittest/test_data_codec.cpp @@ -32,8 +32,8 @@ using namespace milvus; TEST(storage, InsertDataBool) { FixedVector data = {true, false, true, false, true}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::BOOL, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::BOOL, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -62,8 +62,8 @@ TEST(storage, InsertDataBool) { TEST(storage, InsertDataBoolNullable) { FixedVector data = {true, false, false, false, true}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::BOOL, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::BOOL, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -99,8 +99,8 @@ TEST(storage, InsertDataBoolNullable) { TEST(storage, InsertDataInt8) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT8, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT8, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -129,8 +129,8 @@ TEST(storage, InsertDataInt8) { TEST(storage, InsertDataInt8Nullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT8, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT8, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -163,8 +163,8 @@ TEST(storage, InsertDataInt8Nullable) { TEST(storage, InsertDataInt16) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT16, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT16, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -193,8 +193,8 @@ TEST(storage, InsertDataInt16) { TEST(storage, InsertDataInt16Nullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT16, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT16, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -227,8 +227,8 @@ TEST(storage, InsertDataInt16Nullable) { TEST(storage, InsertDataInt32) { FixedVector data = {true, false, true, false, true}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT32, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT32, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -257,8 +257,8 @@ TEST(storage, InsertDataInt32) { TEST(storage, InsertDataInt32Nullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT32, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT32, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -291,8 +291,8 @@ TEST(storage, InsertDataInt32Nullable) { TEST(storage, InsertDataInt64) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT64, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT64, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -321,8 +321,8 @@ TEST(storage, InsertDataInt64) { TEST(storage, InsertDataInt64Nullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::INT64, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::INT64, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -356,8 +356,8 @@ TEST(storage, InsertDataInt64Nullable) { TEST(storage, InsertDataString) { FixedVector data = { "test1", "test2", "test3", "test4", "test5"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VARCHAR, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VARCHAR, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -391,8 +391,8 @@ TEST(storage, InsertDataString) { TEST(storage, InsertDataStringNullable) { FixedVector data = { "test1", "test2", "test3", "test4", "test5"}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::STRING, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::STRING, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -428,8 +428,8 @@ TEST(storage, InsertDataStringNullable) { TEST(storage, InsertDataFloat) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::FLOAT, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::FLOAT, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -458,8 +458,8 @@ TEST(storage, InsertDataFloat) { TEST(storage, InsertDataFloatNullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::FLOAT, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::FLOAT, DataType::NONE, true); std::array valid_data = {0xF3}; field_data->FillFieldData(data.data(), valid_data.data(), data.size(), 0); @@ -491,8 +491,8 @@ TEST(storage, InsertDataFloatNullable) { TEST(storage, InsertDataDouble) { FixedVector data = {1.0, 2.0, 3.0, 4.2, 5.3}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::DOUBLE, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::DOUBLE, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -521,8 +521,8 @@ TEST(storage, InsertDataDouble) { TEST(storage, InsertDataDoubleNullable) { FixedVector data = {1, 2, 3, 4, 5}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::DOUBLE, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::DOUBLE, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -556,8 +556,8 @@ TEST(storage, InsertDataDoubleNullable) { TEST(storage, InsertDataTimestamptz) { FixedVector data = { 1000000000, 2000000000, 3000000000, 400000, 5000}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::TIMESTAMPTZ, false); + auto field_data = milvus::storage::CreateFieldData( + DataType::TIMESTAMPTZ, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -587,8 +587,8 @@ TEST(storage, InsertDataTimestamptz) { TEST(storage, InsertDataTimestamptzNullable) { FixedVector data = { 1000000000, 2000000000, 3000000000, 400000, 5000}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::TIMESTAMPTZ, true); + auto field_data = milvus::storage::CreateFieldData( + DataType::TIMESTAMPTZ, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -623,7 +623,7 @@ TEST(storage, InsertDataFloatVector) { std::vector data = {1, 2, 3, 4, 5, 6, 7, 8}; int DIM = 2; auto field_data = milvus::storage::CreateFieldData( - storage::DataType::VECTOR_FLOAT, false, DIM); + storage::DataType::VECTOR_FLOAT, DataType::NONE, false, DIM); field_data->FillFieldData(data.data(), data.size() / DIM); auto payload_reader = @@ -656,11 +656,12 @@ TEST(storage, InsertDataSparseFloat) { auto n_rows = 100; auto vecs = milvus::segcore::GenerateRandomSparseFloatVector( n_rows, kTestSparseDim, kTestSparseVectorDensity); - auto field_data = milvus::storage::CreateFieldData( - storage::DataType::VECTOR_SPARSE_U32_F32, - false, - kTestSparseDim, - n_rows); + auto field_data = + milvus::storage::CreateFieldData(DataType::VECTOR_SPARSE_U32_F32, + DataType::NONE, + false, + kTestSparseDim, + n_rows); field_data->FillFieldData(vecs.get(), n_rows); auto payload_reader = @@ -702,7 +703,7 @@ TEST(storage, InsertDataBinaryVector) { std::vector data = {1, 2, 3, 4, 5, 6, 7, 8}; int DIM = 16; auto field_data = milvus::storage::CreateFieldData( - storage::DataType::VECTOR_BINARY, false, DIM); + storage::DataType::VECTOR_BINARY, DataType::NONE, false, DIM); field_data->FillFieldData(data.data(), data.size() * 8 / DIM); auto payload_reader = @@ -733,7 +734,7 @@ TEST(storage, InsertDataFloat16Vector) { std::vector data = {1, 2, 3, 4, 5, 6, 7, 8}; int DIM = 2; auto field_data = milvus::storage::CreateFieldData( - storage::DataType::VECTOR_FLOAT16, false, DIM); + storage::DataType::VECTOR_FLOAT16, DataType::NONE, false, DIM); field_data->FillFieldData(data.data(), data.size() / DIM); auto payload_reader = @@ -796,8 +797,8 @@ TEST(storage, InsertDataStringArray) { field_string_data.mutable_string_data()->add_data("test_array5"); auto string_array = Array(field_string_data); FixedVector data = {string_array}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::ARRAY, false); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::ARRAY, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto payload_reader = @@ -842,8 +843,8 @@ TEST(storage, InsertDataStringArrayNullable) { field_string_data.mutable_int_data()->add_data(5); auto int_array = Array(field_int_data); FixedVector data = {string_array, int_array}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::ARRAY, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::ARRAY, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xFD}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -880,8 +881,8 @@ TEST(storage, InsertDataStringArrayNullable) { TEST(storage, InsertDataJsonNullable) { FixedVector data = {Json(), Json(simdjson::padded_string(std::string("A")))}; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::JSON, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::JSON, DataType::NONE, true); uint8_t* valid_data = new uint8_t[1]{0xFC}; field_data->FillFieldData(data.data(), valid_data, data.size(), 0); @@ -909,8 +910,8 @@ TEST(storage, InsertDataJsonNullable) { } TEST(storage, InsertDataJsonFillWithNull) { - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::JSON, true); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::JSON, DataType::NONE, true); int64_t size = 2; uint8_t* valid_data = new uint8_t[1]{0xFC}; field_data->FillFieldData(std::nullopt, size); diff --git a/internal/core/unittest/test_disk_file_manager_test.cpp b/internal/core/unittest/test_disk_file_manager_test.cpp index 68d99ba9d2..e00c22834d 100644 --- a/internal/core/unittest/test_disk_file_manager_test.cpp +++ b/internal/core/unittest/test_disk_file_manager_test.cpp @@ -102,8 +102,8 @@ TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) { auto buf = std::unique_ptr(new uint8_t[file_size]); lcm->Read(file, buf.get(), file_size); - auto index = - milvus::storage::CreateFieldData(storage::DataType::INT8, false); + auto index = milvus::storage::CreateFieldData( + storage::DataType::INT8, DataType::NONE, false); index->FillFieldData(buf.get(), file_size); auto rows = index->get_num_rows(); auto rawData = (uint8_t*)(index->Data()); @@ -370,7 +370,8 @@ auto PrepareInsertData(const int64_t opt_field_data_range) -> std::string { std::vector data = PrepareRawFieldData(opt_field_data_range); - auto field_data = storage::CreateFieldData(DT, false, 1, kEntityCnt); + auto field_data = + storage::CreateFieldData(DT, DataType::NONE, false, 1, kEntityCnt); field_data->FillFieldData(data.data(), kEntityCnt); auto payload_reader = std::make_shared(field_data); @@ -395,7 +396,8 @@ PrepareOptionalField(const std::shared_ptr& file_manager, OptFieldT opt_field; std::vector insert_files; insert_files.emplace_back(insert_file_path); - opt_field[kOptFieldId] = {kOptFieldName, DT, insert_files}; + opt_field[kOptFieldId] = { + kOptFieldName, DT, DataType::NONE, insert_files}; // 添加element_type return opt_field; } @@ -453,8 +455,10 @@ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOptFieldMoreThanOne) { PrepareInsertData(kOptFieldDataRange); OptFieldT opt_fields = PrepareOptionalField(file_manager, insert_file_path); - opt_fields[kOptFieldId + 1] = { - kOptFieldName + "second", DataType::INT64, {insert_file_path}}; + opt_fields[kOptFieldId + 1] = {kOptFieldName + "second", + DataType::INT64, + DataType::NONE, + {insert_file_path}}; // 添加element_type milvus::Config config; config[VEC_OPT_FIELDS] = opt_fields; EXPECT_THROW(file_manager->CacheOptFieldToDisk(config), SegcoreError); diff --git a/internal/core/unittest/test_growing.cpp b/internal/core/unittest/test_growing.cpp index 4c704f1545..8ada12e9da 100644 --- a/internal/core/unittest/test_growing.cpp +++ b/internal/core/unittest/test_growing.cpp @@ -562,10 +562,12 @@ TEST(GrowingTest, SearchVectorArray) { auto schema = std::make_shared(); auto metric_type = knowhere::metric::MAX_SIM; + auto dim = 32; + // Add fields auto int64_field = schema->AddDebugField("int64", DataType::INT64); auto array_vec = schema->AddDebugVectorArrayField( - "array_vec", DataType::VECTOR_FLOAT, 128, metric_type); + "array_vec", DataType::VECTOR_FLOAT, dim, metric_type); schema->set_primary_field_id(int64_field); // Configure segment @@ -577,7 +579,8 @@ TEST(GrowingTest, SearchVectorArray) { {"index_type", knowhere::IndexEnum::INDEX_EMB_LIST_HNSW}, {"metric_type", metric_type}, {"nlist", "128"}}; - std::map type_params = {{"dim", "128"}}; + std::map type_params = { + {"dim", std::to_string(dim)}}; FieldIndexMeta fieldIndexMeta( array_vec, std::move(index_params), std::move(type_params)); std::map fieldMap = {{array_vec, fieldIndexMeta}}; @@ -602,7 +605,6 @@ TEST(GrowingTest, SearchVectorArray) { // Prepare search query int vec_num = 10; // Total number of query vectors - int dim = 128; std::vector query_vec = generate_float_vector(vec_num, dim); // Create query dataset with lims for VectorArray diff --git a/internal/core/unittest/test_hybrid_index.cpp b/internal/core/unittest/test_hybrid_index.cpp index 85cbc888f5..c629ffc539 100644 --- a/internal/core/unittest/test_hybrid_index.cpp +++ b/internal/core/unittest/test_hybrid_index.cpp @@ -115,7 +115,8 @@ class HybridIndexTestV1 : public testing::Test { data_.push_back(x); } - auto field_data = storage::CreateFieldData(type_, nullable_); + auto field_data = + storage::CreateFieldData(type_, DataType::NONE, nullable_); if (nullable_) { valid_data_.reserve(nb_); uint8_t* ptr = new uint8_t[(nb_ + 7) / 8]; diff --git a/internal/core/unittest/test_inverted_index.cpp b/internal/core/unittest/test_inverted_index.cpp index 9abcd75a49..f8c9f63938 100644 --- a/internal/core/unittest/test_inverted_index.cpp +++ b/internal/core/unittest/test_inverted_index.cpp @@ -118,7 +118,7 @@ test_run() { data.push_back(x); } - auto field_data = storage::CreateFieldData(dtype, nullable); + auto field_data = storage::CreateFieldData(dtype, DataType::NONE, nullable); if (nullable) { int byteSize = (nb + 7) / 8; uint8_t* valid_data_ = new uint8_t[byteSize]; @@ -512,7 +512,7 @@ test_string() { } } - auto field_data = storage::CreateFieldData(dtype, nullable); + auto field_data = storage::CreateFieldData(dtype, DataType::NONE, nullable); if (nullable) { int byteSize = (nb + 7) / 8; uint8_t* valid_data_ = new uint8_t[byteSize]; diff --git a/internal/core/unittest/test_json_flat_index.cpp b/internal/core/unittest/test_json_flat_index.cpp index 60346a1922..deb0d30dc4 100644 --- a/internal/core/unittest/test_json_flat_index.cpp +++ b/internal/core/unittest/test_json_flat_index.cpp @@ -89,7 +89,8 @@ class JsonFlatIndexTest : public ::testing::Test { R"({"profile": {"name": {"first": "Charlie", "last": "Williams"}, "team": {"name": "Design", "supervisor": {"name": "Alice"}}, "is_active": true, "employee_id": 1003, "skills": ["python", "javascript"], "scores": [87, 91, 89]}})"}; // Create field data with JSON values - auto field_data = storage::CreateFieldData(DataType::JSON); + auto field_data = + storage::CreateFieldData(DataType::JSON, DataType::NONE); std::vector json_vec; for (const auto& json_str : json_data_) { json_vec.push_back(Json(simdjson::padded_string(json_str))); diff --git a/internal/core/unittest/test_json_stats/test_json_key_stats.cpp b/internal/core/unittest/test_json_stats/test_json_key_stats.cpp index 81a6b71251..3896eae23a 100644 --- a/internal/core/unittest/test_json_stats/test_json_key_stats.cpp +++ b/internal/core/unittest/test_json_stats/test_json_key_stats.cpp @@ -85,7 +85,8 @@ class JsonKeyStatsTest : public ::testing::TestWithParam { segment_id, field_id, index_build_id, index_version}; data_ = std::move(GenerateJsons(size)); - auto field_data = storage::CreateFieldData(DataType::JSON, nullable_); + auto field_data = + storage::CreateFieldData(DataType::JSON, DataType::NONE, nullable_); if (nullable_) { valid_data.reserve(size_); for (size_t i = 0; i < size_; i++) { @@ -377,7 +378,8 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test { void BuildAndUpload() { - auto field_data = storage::CreateFieldData(DataType::JSON, false); + auto field_data = + storage::CreateFieldData(DataType::JSON, DataType::NONE, false); field_data->FillFieldData(data_.data(), data_.size()); auto payload_reader = diff --git a/internal/core/unittest/test_kmeans_clustering.cpp b/internal/core/unittest/test_kmeans_clustering.cpp index ae7504232e..55646686a3 100644 --- a/internal/core/unittest/test_kmeans_clustering.cpp +++ b/internal/core/unittest/test_kmeans_clustering.cpp @@ -194,7 +194,8 @@ test_run() { for (int64_t i = 0; i < nb * dim; ++i) { data_gen[i] = rand(); } - auto field_data = storage::CreateFieldData(dtype, false, dim); + auto field_data = + storage::CreateFieldData(dtype, DataType::NONE, false, dim); field_data->FillFieldData(data_gen.data(), data_gen.size() / dim); auto payload_reader = std::make_shared(field_data); diff --git a/internal/core/unittest/test_ngram_query.cpp b/internal/core/unittest/test_ngram_query.cpp index 80d013922f..61dfd7252d 100644 --- a/internal/core/unittest/test_ngram_query.cpp +++ b/internal/core/unittest/test_ngram_query.cpp @@ -61,7 +61,8 @@ test_ngram_with_data(const boost::container::vector& data, size_t nb = data.size(); - auto field_data = storage::CreateFieldData(DataType::VARCHAR, false); + auto field_data = + storage::CreateFieldData(DataType::VARCHAR, DataType::NONE, false); field_data->FillFieldData(data.data(), data.size()); auto segment = CreateSealedSegment(schema); diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index fe0a3da83a..e81f74e4ff 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -1208,8 +1208,8 @@ TEST(Sealed, BF) { schema, dataset, false, {fake_id.get()}); auto vec_data = GenRandomFloatVecs(N, dim); - auto field_data = - storage::CreateFieldData(DataType::VECTOR_FLOAT, false, dim); + auto field_data = storage::CreateFieldData( + DataType::VECTOR_FLOAT, DataType::NONE, false, dim); field_data->FillFieldData(vec_data.data(), N); auto cm = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); @@ -1272,8 +1272,8 @@ TEST(Sealed, BF_Overflow) { GetExcludedFieldIds(schema, {0, 1, i64_fid.get()})); auto vec_data = GenMaxFloatVecs(N, dim); - auto field_data = - storage::CreateFieldData(DataType::VECTOR_FLOAT, false, dim); + auto field_data = storage::CreateFieldData( + DataType::VECTOR_FLOAT, DataType::NONE, false, dim); field_data->FillFieldData(vec_data.data(), N); auto cm = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); @@ -1577,7 +1577,7 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { //test for int64 std::vector pks = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; auto pk_field_data = - storage::CreateFieldData(DataType::INT64, false, 1, 10); + storage::CreateFieldData(DataType::INT64, DataType::NONE, false, 1, 10); pk_field_data->FillFieldData(pks.data(), N); auto load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1624,7 +1624,7 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { //test for int32 std::vector int32s = {2, 2, 3, 4, 5, 6, 7, 8, 9, 12}; auto int32_field_data = - storage::CreateFieldData(DataType::INT32, false, 1, 10); + storage::CreateFieldData(DataType::INT32, DataType::NONE, false, 1, 10); int32_field_data->FillFieldData(int32s.data(), N); load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1640,7 +1640,7 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { //test for int16 std::vector int16s = {2, 2, 3, 4, 5, 6, 7, 8, 9, 12}; auto int16_field_data = - storage::CreateFieldData(DataType::INT16, false, 1, 10); + storage::CreateFieldData(DataType::INT16, DataType::NONE, false, 1, 10); int16_field_data->FillFieldData(int16s.data(), N); load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1656,7 +1656,7 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { //test for int8 std::vector int8s = {2, 2, 3, 4, 5, 6, 7, 8, 9, 12}; auto int8_field_data = - storage::CreateFieldData(DataType::INT8, false, 1, 10); + storage::CreateFieldData(DataType::INT8, DataType::NONE, false, 1, 10); int8_field_data->FillFieldData(int8s.data(), N); load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1673,7 +1673,7 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { std::vector floats = { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; auto float_field_data = - storage::CreateFieldData(DataType::FLOAT, false, 1, 10); + storage::CreateFieldData(DataType::FLOAT, DataType::NONE, false, 1, 10); float_field_data->FillFieldData(floats.data(), N); load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1689,8 +1689,8 @@ TEST(Sealed, SkipIndexSkipUnaryRange) { // test for double std::vector doubles = { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; - auto double_field_data = - storage::CreateFieldData(DataType::DOUBLE, false, 1, 10); + auto double_field_data = storage::CreateFieldData( + DataType::DOUBLE, DataType::NONE, false, 1, 10); double_field_data->FillFieldData(doubles.data(), N); load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1721,7 +1721,7 @@ TEST(Sealed, SkipIndexSkipBinaryRange) { //test for int64 std::vector pks = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; auto pk_field_data = - storage::CreateFieldData(DataType::INT64, false, 1, 10); + storage::CreateFieldData(DataType::INT64, DataType::NONE, false, 1, 10); pk_field_data->FillFieldData(pks.data(), N); auto load_info = PrepareSingleFieldInsertBinlog(kCollectionID, kPartitionID, @@ -1764,7 +1764,7 @@ TEST(Sealed, SkipIndexSkipUnaryRangeNullable) { std::vector int64s = {1, 2, 3, 4, 5}; std::array valid_data = {0x03}; auto int64s_field_data = - storage::CreateFieldData(DataType::INT64, true, 1, 5); + storage::CreateFieldData(DataType::INT64, DataType::NONE, true, 1, 5); int64s_field_data->FillFieldData(int64s.data(), valid_data.data(), 5, 0); auto load_info = PrepareSingleFieldInsertBinlog(kCollectionID, @@ -1835,7 +1835,7 @@ TEST(Sealed, SkipIndexSkipBinaryRangeNullable) { std::vector int64s = {1, 2, 3, 4, 5}; std::array valid_data = {0x03}; auto int64s_field_data = - storage::CreateFieldData(DataType::INT64, true, 1, 5); + storage::CreateFieldData(DataType::INT64, DataType::NONE, true, 1, 5); int64s_field_data->FillFieldData(int64s.data(), valid_data.data(), 5, 0); auto load_info = PrepareSingleFieldInsertBinlog(kCollectionID, @@ -1876,8 +1876,8 @@ TEST(Sealed, SkipIndexSkipStringRange) { //test for string std::vector strings = {"e", "f", "g", "g", "j"}; - auto string_field_data = - storage::CreateFieldData(DataType::VARCHAR, false, 1, N); + auto string_field_data = storage::CreateFieldData( + DataType::VARCHAR, DataType::NONE, false, 1, N); string_field_data->FillFieldData(strings.data(), N); auto cm = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); @@ -2336,12 +2336,13 @@ TEST(Sealed, SearchVectorArray) { int64_t index_build_id = 4000; int64_t index_version = 4000; int64_t index_id = 5000; + int64_t dim = 32; auto schema = std::make_shared(); - auto metric_type = knowhere::metric::L2; + auto metric_type = knowhere::metric::MAX_SIM; auto int64_field = schema->AddDebugField("int64", DataType::INT64); auto array_vec = schema->AddDebugVectorArrayField( - "array_vec", DataType::VECTOR_FLOAT, 128, metric_type); + "array_vec", DataType::VECTOR_FLOAT, dim, metric_type); schema->set_primary_field_id(int64_field); auto field_meta = milvus::segcore::gen_field_meta(collection_id, @@ -2359,7 +2360,6 @@ TEST(Sealed, SearchVectorArray) { std::make_shared(100000, std::move(filedMap)); int64_t dataset_size = 1000; - int64_t dim = 128; auto emb_list_len = 10; auto dataset = DataGen(schema, dataset_size, 42, 0, 1, emb_list_len); @@ -2372,7 +2372,8 @@ TEST(Sealed, SearchVectorArray) { for (auto& v : vec_array_col) { vector_arrays.push_back(milvus::VectorArray(v)); } - auto field_data = storage::CreateFieldData(DataType::VECTOR_ARRAY, false); + auto field_data = storage::CreateFieldData( + DataType::VECTOR_ARRAY, DataType::VECTOR_FLOAT, false, dim); field_data->FillFieldData(vector_arrays.data(), vector_arrays.size()); // create sealed segment diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index cb2739d2d7..e154a4e0da 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -1168,18 +1168,21 @@ CreateFieldDataFromDataArray(ssize_t raw_count, int64_t dim = 1; FieldDataPtr field_data = nullptr; - auto createFieldData = [&field_data, &raw_count](const void* raw_data, - DataType data_type, - int64_t dim) { - field_data = storage::CreateFieldData(data_type, false, dim); - field_data->FillFieldData(raw_data, raw_count); - }; - auto createNullableFieldData = [&field_data, &raw_count]( + auto element_type = field_meta.get_element_type(); + auto createFieldData = + [&field_data, &raw_count, &element_type]( + const void* raw_data, DataType data_type, int64_t dim) { + field_data = + storage::CreateFieldData(data_type, element_type, false, dim); + field_data->FillFieldData(raw_data, raw_count); + }; + auto createNullableFieldData = [&field_data, &raw_count, &element_type]( const void* raw_data, const bool* raw_valid_data, DataType data_type, int64_t dim) { - field_data = storage::CreateFieldData(data_type, true, dim); + field_data = + storage::CreateFieldData(data_type, element_type, true, dim); int byteSize = (raw_count + 7) / 8; std::vector valid_data(byteSize); for (int i = 0; i < raw_count; i++) { diff --git a/internal/core/unittest/test_vector_array.cpp b/internal/core/unittest/test_vector_array.cpp index df5f48bdbe..69dbb7e059 100644 --- a/internal/core/unittest/test_vector_array.cpp +++ b/internal/core/unittest/test_vector_array.cpp @@ -133,3 +133,77 @@ TEST(VectorArray, TestConstructVectorArray) { // todo: add other vector types } + +TEST(VectorArray, TestConstructorWithData) { + using namespace milvus; + + int N = 10; // number of vectors + int64_t dim = 128; + + // Generate test data + auto data = generate_float_vector(42, N, dim); + + // Test 1: Direct construction from raw float data + { + milvus::VectorArray va(data.data(), N, dim, DataType::VECTOR_FLOAT); + + ASSERT_EQ(va.length(), N); + ASSERT_EQ(va.dim(), dim); + ASSERT_EQ(va.get_element_type(), DataType::VECTOR_FLOAT); + ASSERT_EQ(va.byte_size(), N * dim * sizeof(float)); + + // Verify data integrity + for (int i = 0; i < N; ++i) { + auto vec_data = va.get_data(i); + for (int j = 0; j < dim; ++j) { + ASSERT_FLOAT_EQ(vec_data[j], data[i * dim + j]); + } + } + } + + // Test 2: Compare with protobuf-based constructor + { + // Create via protobuf + milvus::proto::schema::VectorField field_proto; + field_proto.set_dim(dim); + field_proto.mutable_float_vector()->mutable_data()->Add(data.begin(), + data.end()); + milvus::VectorArray va_proto(field_proto); + + // Create via data constructor + milvus::VectorArray va_direct( + data.data(), N, dim, DataType::VECTOR_FLOAT); + + // Both should be equal + ASSERT_EQ(va_proto.length(), va_direct.length()); + ASSERT_EQ(va_proto.dim(), va_direct.dim()); + ASSERT_EQ(va_proto.byte_size(), va_direct.byte_size()); + ASSERT_EQ(va_proto.get_element_type(), va_direct.get_element_type()); + + // Compare data + for (int i = 0; i < N; ++i) { + auto proto_vec = va_proto.get_data(i); + auto direct_vec = va_direct.get_data(i); + for (int j = 0; j < dim; ++j) { + ASSERT_FLOAT_EQ(proto_vec[j], direct_vec[j]); + } + } + } + + // Test 3: Test with edge cases + { + // Single vector + milvus::VectorArray va_single( + data.data(), 1, dim, DataType::VECTOR_FLOAT); + ASSERT_EQ(va_single.length(), 1); + ASSERT_EQ(va_single.byte_size(), dim * sizeof(float)); + + // Small dimension + int64_t small_dim = 4; + auto small_data = generate_float_vector(123, 5, small_dim); + milvus::VectorArray va_small( + small_data.data(), 5, small_dim, DataType::VECTOR_FLOAT); + ASSERT_EQ(va_small.length(), 5); + ASSERT_EQ(va_small.dim(), small_dim); + } +} diff --git a/internal/core/unittest/test_vector_array_chunk.cpp b/internal/core/unittest/test_vector_array_chunk.cpp new file mode 100644 index 0000000000..dde4d22c99 --- /dev/null +++ b/internal/core/unittest/test_vector_array_chunk.cpp @@ -0,0 +1,237 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include +#include +#include +#include +#include +#include + +#include "common/ChunkWriter.h" +#include "common/Chunk.h" +#include "common/VectorArray.h" +#include "common/Types.h" +#include "pb/schema.pb.h" +#include "test_utils/DataGen.h" + +using namespace milvus; + +class VectorArrayChunkTest : public ::testing::Test { + protected: + std::vector + generateFloatVector(int64_t seed, int64_t N, int64_t dim) { + std::vector result(dim * N); + std::default_random_engine gen(seed); + std::normal_distribution dist(0.0f, 1.0f); + + for (int64_t i = 0; i < dim * N; ++i) { + result[i] = dist(gen); + } + return result; + } + + std::shared_ptr + createFloatVectorListArray(const std::vector& data, + const std::vector& offsets) { + auto float_builder = std::make_shared(); + auto list_builder = std::make_shared( + arrow::default_memory_pool(), float_builder); + + arrow::Status ast; + for (size_t i = 0; i < offsets.size() - 1; ++i) { + ast = list_builder->Append(); + assert(ast.ok()); + int32_t start = offsets[i]; + int32_t end = offsets[i + 1]; + + for (int32_t j = start; j < end; ++j) { + float_builder->Append(data[j]); + } + } + + std::shared_ptr array; + ast = list_builder->Finish(&array); + assert(ast.ok()); + return std::static_pointer_cast(array); + } +}; + +TEST_F(VectorArrayChunkTest, TestWriteFloatVectorArray) { + // Test parameters + const int64_t dim = 128; + const int num_rows = 100; + const int vectors_per_row = 5; // Each row contains 5 vectors + + // Generate test data + std::vector all_data; + std::vector offsets = {0}; + + for (int row = 0; row < num_rows; ++row) { + auto row_data = generateFloatVector(row, vectors_per_row, dim); + all_data.insert(all_data.end(), row_data.begin(), row_data.end()); + offsets.push_back(offsets.back() + vectors_per_row * dim); + } + + // Create Arrow ListArray + auto list_array = createFloatVectorListArray(all_data, offsets); + arrow::ArrayVector array_vec = {list_array}; + + // Test VectorArrayChunkWriter + VectorArrayChunkWriter writer(dim, DataType::VECTOR_FLOAT); + writer.write(array_vec); + + auto chunk = writer.finish(); + auto vector_array_chunk = static_cast(chunk.get()); + + // Verify results + EXPECT_EQ(vector_array_chunk->RowNums(), num_rows); + + // Verify data integrity using View method + for (int row = 0; row < num_rows; ++row) { + auto view = vector_array_chunk->View(row); + + // Verify by converting back to VectorFieldProto and checking + auto proto = view.output_data(); + EXPECT_EQ(proto.dim(), dim); + EXPECT_EQ(proto.float_vector().data_size(), vectors_per_row * dim); + + const float* expected = all_data.data() + row * vectors_per_row * dim; + for (int i = 0; i < vectors_per_row * dim; ++i) { + EXPECT_FLOAT_EQ(proto.float_vector().data(i), expected[i]); + } + } +} + +TEST_F(VectorArrayChunkTest, TestWriteMultipleBatches) { + const int64_t dim = 64; + const int batch_size = 50; + const int num_batches = 3; + const int vectors_per_row = 3; + + arrow::ArrayVector array_vec; + std::vector> all_batch_data; + + // Create multiple batches + for (int batch = 0; batch < num_batches; ++batch) { + std::vector batch_data; + std::vector batch_offsets = {0}; + + for (int row = 0; row < batch_size; ++row) { + auto row_data = + generateFloatVector(batch * 1000 + row, vectors_per_row, dim); + batch_data.insert( + batch_data.end(), row_data.begin(), row_data.end()); + batch_offsets.push_back(batch_offsets.back() + + vectors_per_row * dim); + } + + all_batch_data.push_back(batch_data); + array_vec.push_back( + createFloatVectorListArray(batch_data, batch_offsets)); + } + + // Write using VectorArrayChunkWriter + VectorArrayChunkWriter writer(dim, DataType::VECTOR_FLOAT); + writer.write(array_vec); + + auto chunk = writer.finish(); + auto vector_array_chunk = static_cast(chunk.get()); + + // Verify total rows + EXPECT_EQ(vector_array_chunk->RowNums(), batch_size * num_batches); + + // Verify data from each batch using View method + for (int batch = 0; batch < num_batches; ++batch) { + for (int row = 0; row < batch_size; ++row) { + int global_row = batch * batch_size + row; + auto view = vector_array_chunk->View(global_row); + + // Verify by converting back to VectorFieldProto and checking + auto proto = view.output_data(); + EXPECT_EQ(proto.dim(), dim); + EXPECT_EQ(proto.float_vector().data_size(), vectors_per_row * dim); + + const float* expected = + all_batch_data[batch].data() + row * vectors_per_row * dim; + for (int i = 0; i < vectors_per_row * dim; ++i) { + EXPECT_FLOAT_EQ(proto.float_vector().data(i), expected[i]); + } + } + } +} + +TEST_F(VectorArrayChunkTest, TestWriteWithMmap) { + const int64_t dim = 128; + const int num_rows = 100; + const int vectors_per_row = 4; + + // Create temp file path + std::string temp_file = + "/tmp/test_vector_array_chunk_" + + std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count()); + + // Generate test data + std::vector all_data; + std::vector offsets = {0}; + + for (int row = 0; row < num_rows; ++row) { + auto row_data = generateFloatVector(row, vectors_per_row, dim); + all_data.insert(all_data.end(), row_data.begin(), row_data.end()); + offsets.push_back(offsets.back() + vectors_per_row * dim); + } + + auto list_array = createFloatVectorListArray(all_data, offsets); + arrow::ArrayVector array_vec = {list_array}; + + // Write with mmap + VectorArrayChunkWriter writer(dim, DataType::VECTOR_FLOAT, temp_file); + writer.write(array_vec); + + auto chunk = writer.finish(); + auto vector_array_chunk = static_cast(chunk.get()); + + // Verify mmap write + EXPECT_EQ(vector_array_chunk->RowNums(), num_rows); + + for (int row = 0; row < num_rows; ++row) { + auto view = vector_array_chunk->View(row); + + // Verify by converting back to VectorFieldProto and checking + auto proto = view.output_data(); + EXPECT_EQ(proto.dim(), dim); + EXPECT_EQ(proto.float_vector().data_size(), vectors_per_row * dim); + + const float* expected = all_data.data() + row * vectors_per_row * dim; + for (int i = 0; i < vectors_per_row * dim; ++i) { + EXPECT_FLOAT_EQ(proto.float_vector().data(i), expected[i]); + } + } + + // Clean up + std::remove(temp_file.c_str()); +} + +TEST_F(VectorArrayChunkTest, TestEmptyVectorArray) { + const int64_t dim = 128; + + arrow::ArrayVector array_vec; + + VectorArrayChunkWriter writer(dim, DataType::VECTOR_FLOAT); + writer.write(array_vec); + + auto chunk = writer.finish(); + auto vector_array_chunk = static_cast(chunk.get()); + + EXPECT_EQ(vector_array_chunk->RowNums(), 0); +} diff --git a/internal/core/unittest/test_vector_array_storage_v2.cpp b/internal/core/unittest/test_vector_array_storage_v2.cpp index c6450b1820..ac90523a7a 100644 --- a/internal/core/unittest/test_vector_array_storage_v2.cpp +++ b/internal/core/unittest/test_vector_array_storage_v2.cpp @@ -150,31 +150,43 @@ class TestVectorArrayStorageV2 : public testing::Test { EXPECT_TRUE(status.ok()); arrays.push_back(array); } else { - // vector array - arrow::BinaryBuilder builder; + // vector array - using ListArray + // Get field meta to determine element type + auto vector_array_field_id = fields_["vector_array"]; + auto& field_meta = + schema_->operator[](vector_array_field_id); + auto element_type = field_meta.get_element_type(); + + // Create appropriate value builder based on element type + std::shared_ptr value_builder; + if (element_type == DataType::VECTOR_FLOAT) { + value_builder = std::make_shared(); + } else { + FAIL() << "Unsupported element type for VECTOR_ARRAY " + "in test"; + } + + auto list_builder = std::make_shared( + arrow::default_memory_pool(), value_builder); for (int row = 0; row < test_data_count_; row++) { - milvus::proto::schema::VectorField - field_float_vector_array; - field_float_vector_array.set_dim(DIM); - - auto data = generate_float_vector(10, DIM); - field_float_vector_array.mutable_float_vector() - ->mutable_data() - ->Add(data.begin(), data.end()); - - std::string serialized_data; - bool success = - field_float_vector_array.SerializeToString( - &serialized_data); - EXPECT_TRUE(success); - - auto status = builder.Append(serialized_data); + // Each row contains 10 vectors of dimension DIM + auto status = list_builder->Append(); EXPECT_TRUE(status.ok()); + + // Generate 10 vectors for this row + auto data = generate_float_vector(10, DIM); + auto float_builder = + std::static_pointer_cast( + value_builder); + for (const auto& value : data) { + status = float_builder->Append(value); + EXPECT_TRUE(status.ok()); + } } std::shared_ptr array; - auto status = builder.Finish(&array); + auto status = list_builder->Finish(&array); EXPECT_TRUE(status.ok()); arrays.push_back(array); } @@ -297,6 +309,7 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndex) { test_data_count_ * chunk_num_; // Important: set row count config[STORAGE_VERSION_KEY] = 2; // Use storage v2 config[DATA_TYPE_KEY] = DataType::VECTOR_ARRAY; + config[ELEMENT_TYPE_KEY] = DataType::VECTOR_FLOAT; // For storage v2, we need to provide segment insert files instead of individual binlog files milvus::SegmentInsertFiles segment_insert_files; diff --git a/internal/datacoord/task_index.go b/internal/datacoord/task_index.go index 4f3149f49b..1541e353ca 100644 --- a/internal/datacoord/task_index.go +++ b/internal/datacoord/task_index.go @@ -346,10 +346,11 @@ func (it *indexBuildTask) prepareOptionalFields(ctx context.Context, collectionI partitionKeyField, _ := typeutil.GetPartitionKeyFieldSchema(schema) if partitionKeyField != nil && typeutil.IsFieldDataTypeSupportMaterializedView(partitionKeyField) { optionalFields = append(optionalFields, &indexpb.OptionalFieldInfo{ - FieldID: partitionKeyField.FieldID, - FieldName: partitionKeyField.Name, - FieldType: int32(partitionKeyField.DataType), - DataIds: getBinLogIDs(segment, partitionKeyField.FieldID), + FieldID: partitionKeyField.FieldID, + FieldName: partitionKeyField.Name, + FieldType: int32(partitionKeyField.DataType), + ElementType: int32(partitionKeyField.GetElementType()), + DataIds: getBinLogIDs(segment, partitionKeyField.FieldID), }) iso, isoErr := common.IsPartitionKeyIsolationPropEnabled(collectionInfo.Properties) diff --git a/internal/datanode/index/task_index.go b/internal/datanode/index/task_index.go index 160f0eddb8..b2854592d3 100644 --- a/internal/datanode/index/task_index.go +++ b/internal/datanode/index/task_index.go @@ -268,10 +268,11 @@ func (it *indexBuildTask) Execute(ctx context.Context) error { optFields := make([]*indexcgopb.OptionalFieldInfo, 0, len(it.req.GetOptionalScalarFields())) for _, optField := range it.req.GetOptionalScalarFields() { optFields = append(optFields, &indexcgopb.OptionalFieldInfo{ - FieldID: optField.GetFieldID(), - FieldName: optField.GetFieldName(), - FieldType: optField.GetFieldType(), - DataPaths: optField.GetDataPaths(), + FieldID: optField.GetFieldID(), + FieldName: optField.GetFieldName(), + FieldType: optField.GetFieldType(), + ElementType: optField.GetElementType(), + DataPaths: optField.GetDataPaths(), }) } diff --git a/internal/storage/arrow_util.go b/internal/storage/arrow_util.go index f3adda90fb..9b0cbad83d 100644 --- a/internal/storage/arrow_util.go +++ b/internal/storage/arrow_util.go @@ -232,6 +232,41 @@ func appendValueAt(builder array.Builder, a arrow.Array, idx int, defaultValue * b.Append(val) return uint64(len(val)), nil } + case *array.ListBuilder: + // Handle ListBuilder for ArrayOfVector type + if a == nil { + b.AppendNull() + return 0, nil + } + la, ok := a.(*array.List) + if !ok { + return 0, fmt.Errorf("invalid value type %T, expect %T", a.DataType(), builder.Type()) + } + if la.IsNull(idx) { + b.AppendNull() + return 0, nil + } + + start, end := la.ValueOffsets(idx) + b.Append(true) + + valuesArray := la.ListValues() + valueBuilder := b.ValueBuilder() + + var totalSize uint64 = 0 + switch vb := valueBuilder.(type) { + case *array.Float32Builder: + if floatArray, ok := valuesArray.(*array.Float32); ok { + for i := start; i < end; i++ { + vb.Append(floatArray.Value(int(i))) + totalSize += 4 + } + } + default: + return 0, fmt.Errorf("unsupported value builder type in ListBuilder: %T", valueBuilder) + } + + return totalSize, nil default: return 0, fmt.Errorf("unsupported builder type: %T", builder) } @@ -247,7 +282,12 @@ func GenerateEmptyArrayFromSchema(schema *schemapb.FieldSchema, numRows int) (ar return nil, merr.WrapErrServiceInternal(fmt.Sprintf("missing field data %s", schema.Name)) } dim, _ := typeutil.GetDim(schema) - builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[schema.GetDataType()].arrowType(int(dim))) // serdeEntry[schema.GetDataType()].newBuilder() + + elementType := schemapb.DataType_None + if schema.GetDataType() == schemapb.DataType_ArrayOfVector { + elementType = schema.GetElementType() + } + builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[schema.GetDataType()].arrowType(int(dim), elementType)) // serdeEntry[schema.GetDataType()].newBuilder() if schema.GetDefaultValue() != nil { switch schema.GetDataType() { case schemapb.DataType_Bool: @@ -376,7 +416,13 @@ func NewRecordBuilder(schema *schemapb.CollectionSchema) *RecordBuilder { builders := make([]array.Builder, len(fields)) for i, field := range fields { dim, _ := typeutil.GetDim(field) - builders[i] = array.NewBuilder(memory.DefaultAllocator, serdeMap[field.DataType].arrowType(int(dim))) + + elementType := schemapb.DataType_None + if field.DataType == schemapb.DataType_ArrayOfVector { + elementType = field.GetElementType() + } + arrowType := serdeMap[field.DataType].arrowType(int(dim), elementType) + builders[i] = array.NewBuilder(memory.DefaultAllocator, arrowType) } return &RecordBuilder{ diff --git a/internal/storage/arrow_util_test.go b/internal/storage/arrow_util_test.go index c08a700b02..c17f31dbf0 100644 --- a/internal/storage/arrow_util_test.go +++ b/internal/storage/arrow_util_test.go @@ -209,7 +209,7 @@ func TestGenerateEmptyArray(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, rowNum, a.Len()) for i := range rowNum { - value, ok := serdeMap[tc.field.DataType].deserialize(a, i, false) + value, ok := serdeMap[tc.field.DataType].deserialize(a, i, schemapb.DataType_None, 0, false) assert.True(t, a.IsValid(i)) assert.True(t, ok) assert.Equal(t, tc.expectValue, value) diff --git a/internal/storage/binlog_test.go b/internal/storage/binlog_test.go index e6f58a8db6..4f3c196fc4 100644 --- a/internal/storage/binlog_test.go +++ b/internal/storage/binlog_test.go @@ -42,21 +42,21 @@ func TestInsertBinlog(t *testing.T) { e1, err := w.NextInsertEventWriter() assert.NoError(t, err) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = e1.AddDataToPayload([]int32{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int32{4, 5, 6}, nil) assert.Error(t, err) - err = e1.AddDataToPayload([]int64{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) e1.SetEventTimestamp(100, 200) e2, err := w.NextInsertEventWriter() assert.NoError(t, err) - err = e2.AddDataToPayload([]int64{7, 8, 9}, nil) + err = e2.AddDataToPayloadForUT([]int64{7, 8, 9}, nil) assert.NoError(t, err) - err = e2.AddDataToPayload([]bool{true, false, true}, nil) + err = e2.AddDataToPayloadForUT([]bool{true, false, true}, nil) assert.Error(t, err) - err = e2.AddDataToPayload([]int64{10, 11, 12}, nil) + err = e2.AddDataToPayloadForUT([]int64{10, 11, 12}, nil) assert.NoError(t, err) e2.SetEventTimestamp(300, 400) @@ -293,21 +293,21 @@ func TestDeleteBinlog(t *testing.T) { e1, err := w.NextDeleteEventWriter() assert.NoError(t, err) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = e1.AddDataToPayload([]int32{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int32{4, 5, 6}, nil) assert.Error(t, err) - err = e1.AddDataToPayload([]int64{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) e1.SetEventTimestamp(100, 200) e2, err := w.NextDeleteEventWriter() assert.NoError(t, err) - err = e2.AddDataToPayload([]int64{7, 8, 9}, nil) + err = e2.AddDataToPayloadForUT([]int64{7, 8, 9}, nil) assert.NoError(t, err) - err = e2.AddDataToPayload([]bool{true, false, true}, nil) + err = e2.AddDataToPayloadForUT([]bool{true, false, true}, nil) assert.Error(t, err) - err = e2.AddDataToPayload([]int64{10, 11, 12}, nil) + err = e2.AddDataToPayloadForUT([]int64{10, 11, 12}, nil) assert.NoError(t, err) e2.SetEventTimestamp(300, 400) @@ -831,11 +831,11 @@ func TestNewBinlogReaderError(t *testing.T) { e1, err := w.NextInsertEventWriter() assert.NoError(t, err) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = e1.AddDataToPayload([]int32{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int32{4, 5, 6}, nil) assert.Error(t, err) - err = e1.AddDataToPayload([]int64{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) e1.SetEventTimestamp(100, 200) @@ -899,7 +899,7 @@ func TestInsertBinlogWriterCloseError(t *testing.T) { sizeTotal := 2000000 insertWriter.baseBinlogWriter.descriptorEventData.AddExtra(originalSizeKey, fmt.Sprintf("%v", sizeTotal)) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) e1.SetEventTimestamp(100, 200) insertWriter.SetEventTimeStamp(1000, 2000) @@ -918,7 +918,7 @@ func TestDeleteBinlogWriteCloseError(t *testing.T) { assert.NoError(t, err) sizeTotal := 2000000 deleteWriter.baseBinlogWriter.descriptorEventData.AddExtra(originalSizeKey, fmt.Sprintf("%v", sizeTotal)) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) e1.SetEventTimestamp(100, 200) deleteWriter.SetEventTimeStamp(1000, 2000) diff --git a/internal/storage/data_codec.go b/internal/storage/data_codec.go index ee9e9aeceb..b8c21b4a48 100644 --- a/internal/storage/data_codec.go +++ b/internal/storage/data_codec.go @@ -303,6 +303,10 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique } payloadWriterOpts = append(payloadWriterOpts, WithDim(int(dim))) } + + if field.DataType == schemapb.DataType_ArrayOfVector { + payloadWriterOpts = append(payloadWriterOpts, WithElementType(field.GetElementType())) + } eventWriter, err := writer.NextInsertEventWriter(payloadWriterOpts...) if err != nil { writer.Close() @@ -458,11 +462,9 @@ func AddFieldDataToPayload(eventWriter *insertEventWriter, dataType schemapb.Dat return err } case schemapb.DataType_ArrayOfVector: - // todo(SpadeA): optimize the serialization method - for _, singleArray := range singleData.(*VectorArrayFieldData).Data { - if err = eventWriter.AddOneVectorArrayToPayload(singleArray); err != nil { - return err - } + vectorArrayData := singleData.(*VectorArrayFieldData) + if err = eventWriter.AddVectorArrayFieldDataToPayload(vectorArrayData); err != nil { + return err } default: return fmt.Errorf("undefined data type %d", dataType) diff --git a/internal/storage/event_test.go b/internal/storage/event_test.go index fe4950904f..fc363e9fed 100644 --- a/internal/storage/event_test.go +++ b/internal/storage/event_test.go @@ -199,13 +199,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Bool, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]bool{true, false, true}, nil) + return w.AddDataToPayloadForUT([]bool{true, false, true}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]bool{false, true, false}, nil) + return w.AddDataToPayloadForUT([]bool{false, true, false}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []bool{true, false, true, false, true, false}) }) @@ -215,13 +215,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Int8, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int8{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]int8{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int8{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int8{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []int8{1, 2, 3, 4, 5, 6}) }) @@ -231,13 +231,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Int16, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int16{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]int16{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int16{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int16{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []int16{1, 2, 3, 4, 5, 6}) }) @@ -247,13 +247,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Int32, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int32{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]int32{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int32{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int32{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []int32{1, 2, 3, 4, 5, 6}) }) @@ -263,13 +263,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Int64, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int64{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int64{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []int64{1, 2, 3, 4, 5, 6}) }) @@ -279,13 +279,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Float, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float32{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]float32{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float32{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]float32{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []float32{1, 2, 3, 4, 5, 6}) }) @@ -295,13 +295,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_Double, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float64{1, 2, 3}, nil) + return w.AddDataToPayloadForUT([]float64{1, 2, 3}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float64{4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]float64{4, 5, 6}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5}, nil) }, []float64{1, 2, 3, 4, 5, 6}) }) @@ -311,13 +311,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_BinaryVector, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]byte{1, 2, 3, 4}, nil) + return w.AddDataToPayloadForUT([]byte{1, 2, 3, 4}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]byte{5, 6, 7, 8}, nil) + return w.AddDataToPayloadForUT([]byte{5, 6, 7, 8}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5, 6}, nil) }, []byte{1, 2, 3, 4, 5, 6, 7, 8}) }) @@ -327,13 +327,13 @@ func TestInsertEvent(t *testing.T) { assert.NoError(t, err) insertT(t, schemapb.DataType_FloatVector, w, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float32{1, 2, 3, 4}, nil) + return w.AddDataToPayloadForUT([]float32{1, 2, 3, 4}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]float32{5, 6, 7, 8}, nil) + return w.AddDataToPayloadForUT([]float32{5, 6, 7, 8}, nil) }, func(w *insertEventWriter) error { - return w.AddDataToPayload([]int{1, 2, 3, 4, 5, 6}, nil) + return w.AddDataToPayloadForUT([]int{1, 2, 3, 4, 5, 6}, nil) }, []float32{1, 2, 3, 4, 5, 6, 7, 8}) }) @@ -342,13 +342,13 @@ func TestInsertEvent(t *testing.T) { w, err := newInsertEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -396,13 +396,13 @@ func TestDeleteEvent(t *testing.T) { w, err := newDeleteEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -456,11 +456,11 @@ func TestCreateCollectionEvent(t *testing.T) { w, err := newCreateCollectionEventWriter(schemapb.DataType_Int64) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int{4, 5, 6}, nil) assert.Error(t, err) - err = w.AddDataToPayload([]int64{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) err = w.Finish() assert.NoError(t, err) @@ -498,13 +498,13 @@ func TestCreateCollectionEvent(t *testing.T) { w, err := newCreateCollectionEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -558,11 +558,11 @@ func TestDropCollectionEvent(t *testing.T) { w, err := newDropCollectionEventWriter(schemapb.DataType_Int64) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int{4, 5, 6}, nil) assert.Error(t, err) - err = w.AddDataToPayload([]int64{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) err = w.Finish() assert.NoError(t, err) @@ -600,13 +600,13 @@ func TestDropCollectionEvent(t *testing.T) { w, err := newDropCollectionEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -660,11 +660,11 @@ func TestCreatePartitionEvent(t *testing.T) { w, err := newCreatePartitionEventWriter(schemapb.DataType_Int64) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int{4, 5, 6}, nil) assert.Error(t, err) - err = w.AddDataToPayload([]int64{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) err = w.Finish() assert.NoError(t, err) @@ -702,13 +702,13 @@ func TestCreatePartitionEvent(t *testing.T) { w, err := newCreatePartitionEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -762,11 +762,11 @@ func TestDropPartitionEvent(t *testing.T) { w, err := newDropPartitionEventWriter(schemapb.DataType_Int64) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int{4, 5, 6}, nil) assert.Error(t, err) - err = w.AddDataToPayload([]int64{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) err = w.Finish() assert.NoError(t, err) @@ -804,13 +804,13 @@ func TestDropPartitionEvent(t *testing.T) { w, err := newDropPartitionEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.AddOneStringToPayload("567890", true) assert.NoError(t, err) err = w.AddOneStringToPayload("abcdefg", true) assert.NoError(t, err) - err = w.AddDataToPayload([]int{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int{1, 2, 3}, nil) assert.Error(t, err) err = w.Finish() assert.NoError(t, err) @@ -1104,7 +1104,7 @@ func TestEventClose(t *testing.T) { w, err := newInsertEventWriter(schemapb.DataType_String) assert.NoError(t, err) w.SetEventTimestamp(tsoutil.ComposeTS(10, 0), tsoutil.ComposeTS(100, 0)) - err = w.AddDataToPayload("1234", nil) + err = w.AddDataToPayloadForUT("1234", nil) assert.NoError(t, err) err = w.Finish() assert.NoError(t, err) diff --git a/internal/storage/payload.go b/internal/storage/payload.go index 228a6fada3..28546c701b 100644 --- a/internal/storage/payload.go +++ b/internal/storage/payload.go @@ -26,7 +26,7 @@ import ( // PayloadWriterInterface abstracts PayloadWriter type PayloadWriterInterface interface { - AddDataToPayload(any, []bool) error + AddDataToPayloadForUT(any, []bool) error AddBoolToPayload([]bool, []bool) error AddByteToPayload([]byte, []bool) error AddInt8ToPayload([]int8, []bool) error @@ -45,7 +45,7 @@ type PayloadWriterInterface interface { AddBFloat16VectorToPayload([]byte, int) error AddSparseFloatVectorToPayload(*SparseFloatVectorFieldData) error AddInt8VectorToPayload([]int8, int) error - AddOneVectorArrayToPayload(*schemapb.VectorField) error + AddVectorArrayFieldDataToPayload(*VectorArrayFieldData) error FinishPayloadWriter() error GetPayloadBufferFromWriter() ([]byte, error) GetPayloadLengthFromWriter() (int, error) diff --git a/internal/storage/payload_reader.go b/internal/storage/payload_reader.go index d7d72243c3..0b66169429 100644 --- a/internal/storage/payload_reader.go +++ b/internal/storage/payload_reader.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strconv" "time" "github.com/apache/arrow/go/v17/arrow" @@ -28,6 +29,10 @@ type PayloadReader struct { colType schemapb.DataType numRows int64 nullable bool + // For VectorArray type + elementType schemapb.DataType + // For VectorArray type + dim int64 } var _ PayloadReaderInterface = (*PayloadReader)(nil) @@ -40,7 +45,73 @@ func NewPayloadReader(colType schemapb.DataType, buf []byte, nullable bool) (*Pa if err != nil { return nil, err } - return &PayloadReader{reader: parquetReader, colType: colType, numRows: parquetReader.NumRows(), nullable: nullable}, nil + + reader := &PayloadReader{ + reader: parquetReader, + colType: colType, + numRows: parquetReader.NumRows(), + nullable: nullable, + } + + if colType == schemapb.DataType_ArrayOfVector { + arrowReader, err := pqarrow.NewFileReader(parquetReader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + if err != nil { + return nil, fmt.Errorf("failed to create arrow reader for VectorArray: %w", err) + } + + arrowSchema, err := arrowReader.Schema() + if err != nil { + return nil, fmt.Errorf("failed to get arrow schema for VectorArray: %w", err) + } + + if arrowSchema.NumFields() != 1 { + return nil, fmt.Errorf("VectorArray should have exactly 1 field, got %d", arrowSchema.NumFields()) + } + + field := arrowSchema.Field(0) + if !field.HasMetadata() { + return nil, errors.New("VectorArray field is missing metadata") + } + + metadata := field.Metadata + + elementTypeStr, ok := metadata.GetValue("elementType") + if !ok { + return nil, errors.New("VectorArray metadata missing required 'elementType' field") + } + elementTypeInt, err := strconv.ParseInt(elementTypeStr, 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid elementType in VectorArray metadata: %s", elementTypeStr) + } + + elementType := schemapb.DataType(elementTypeInt) + switch elementType { + case schemapb.DataType_FloatVector, + schemapb.DataType_BinaryVector, + schemapb.DataType_Float16Vector, + schemapb.DataType_BFloat16Vector, + schemapb.DataType_Int8Vector, + schemapb.DataType_SparseFloatVector: + reader.elementType = elementType + default: + return nil, fmt.Errorf("invalid vector type for VectorArray: %s", elementType.String()) + } + + dimStr, ok := metadata.GetValue("dim") + if !ok { + return nil, errors.New("VectorArray metadata missing required 'dim' field") + } + dimVal, err := strconv.ParseInt(dimStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid dim in VectorArray metadata: %s", dimStr) + } + if dimVal <= 0 { + return nil, fmt.Errorf("VectorArray dim must be positive, got %d", dimVal) + } + reader.dim = dimVal + } + + return reader, nil } // GetDataFromPayload returns data,length from payload, returns err if failed @@ -458,15 +529,7 @@ func (r *PayloadReader) GetVectorArrayFromPayload() ([]*schemapb.VectorField, er return nil, merr.WrapErrParameterInvalidMsg(fmt.Sprintf("failed to get vector from datatype %v", r.colType.String())) } - value, err := readByteAndConvert(r, func(bytes parquet.ByteArray) *schemapb.VectorField { - v := &schemapb.VectorField{} - proto.Unmarshal(bytes, v) - return v - }) - if err != nil { - return nil, err - } - return value, nil + return readVectorArrayFromListArray(r) } func (r *PayloadReader) GetJSONFromPayload() ([][]byte, []bool, error) { @@ -509,6 +572,85 @@ func (r *PayloadReader) GetArrowRecordReader() (pqarrow.RecordReader, error) { return rr, nil } +// readVectorArrayFromListArray reads VectorArray data stored as Arrow ListArray +func readVectorArrayFromListArray(r *PayloadReader) ([]*schemapb.VectorField, error) { + arrowReader, err := pqarrow.NewFileReader(r.reader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + if err != nil { + return nil, err + } + defer arrowReader.ParquetReader().Close() + + // Read all row groups + table, err := arrowReader.ReadTable(context.Background()) + if err != nil { + return nil, err + } + defer table.Release() + + if table.NumCols() != 1 { + return nil, fmt.Errorf("expected 1 column, got %d", table.NumCols()) + } + + column := table.Column(0) + if column.Len() == 0 { + return []*schemapb.VectorField{}, nil + } + + result := make([]*schemapb.VectorField, 0, int(r.numRows)) + + elementType := r.elementType + dim := r.dim + for _, chunk := range column.Data().Chunks() { + listArray, ok := chunk.(*array.List) + if !ok { + return nil, fmt.Errorf("expected ListArray, got %T", chunk) + } + + valuesArray := listArray.ListValues() + switch elementType { + case schemapb.DataType_FloatVector: + floatArray, ok := valuesArray.(*array.Float32) + if !ok { + return nil, fmt.Errorf("expected Float32 array for FloatVector, got %T", valuesArray) + } + + // Process each row which contains multiple vectors + for i := 0; i < listArray.Len(); i++ { + if listArray.IsNull(i) { + return nil, fmt.Errorf("null value in VectorArray") + } + + start, end := listArray.ValueOffsets(i) + vectorData := make([]float32, end-start) + copy(vectorData, floatArray.Float32Values()[start:end]) + + vectorField := &schemapb.VectorField{ + Dim: dim, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: vectorData, + }, + }, + } + result = append(result, vectorField) + } + + case schemapb.DataType_BinaryVector: + return nil, fmt.Errorf("BinaryVector in VectorArray not implemented yet") + case schemapb.DataType_Float16Vector: + return nil, fmt.Errorf("Float16Vector in VectorArray not implemented yet") + case schemapb.DataType_BFloat16Vector: + return nil, fmt.Errorf("BFloat16Vector in VectorArray not implemented yet") + case schemapb.DataType_Int8Vector: + return nil, fmt.Errorf("Int8Vector in VectorArray not implemented yet") + default: + return nil, fmt.Errorf("unsupported element type in VectorArray: %s", elementType.String()) + } + } + + return result, nil +} + func readNullableByteAndConvert[T any](r *PayloadReader, convert func([]byte) T) ([]T, []bool, error) { values := make([][]byte, r.numRows) validData := make([]bool, r.numRows) diff --git a/internal/storage/payload_test.go b/internal/storage/payload_test.go index a8b4918e9f..25c94cacb3 100644 --- a/internal/storage/payload_test.go +++ b/internal/storage/payload_test.go @@ -38,7 +38,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddBoolToPayload([]bool{false, false, false, false}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]bool{false, false, false, false}, nil) + err = w.AddDataToPayloadForUT([]bool{false, false, false, false}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -75,7 +75,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddInt8ToPayload([]int8{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int8{4, 5, 6}, nil) + err = w.AddDataToPayloadForUT([]int8{4, 5, 6}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -115,7 +115,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddInt16ToPayload([]int16{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int16{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int16{1, 2, 3}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -153,7 +153,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddInt32ToPayload([]int32{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int32{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int32{1, 2, 3}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -192,7 +192,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddInt64ToPayload([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -231,7 +231,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddFloatToPayload([]float32{1.0, 2.0, 3.0}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]float32{1.0, 2.0, 3.0}, nil) + err = w.AddDataToPayloadForUT([]float32{1.0, 2.0, 3.0}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -270,7 +270,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddDoubleToPayload([]float64{1.0, 2.0, 3.0}, nil) assert.NoError(t, err) - err = w.AddDataToPayload([]float64{1.0, 2.0, 3.0}, nil) + err = w.AddDataToPayloadForUT([]float64{1.0, 2.0, 3.0}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -313,7 +313,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { assert.NoError(t, err) err = w.AddOneStringToPayload("hello2", true) assert.NoError(t, err) - err = w.AddDataToPayload("hello3", nil) + err = w.AddDataToPayloadForUT("hello3", nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -379,7 +379,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { }, }, true) assert.NoError(t, err) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{7, 8}, @@ -433,7 +433,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { assert.NoError(t, err) err = w.AddOneJSONToPayload([]byte(`{"3":"3"}`), true) assert.NoError(t, err) - err = w.AddDataToPayload([]byte(`{"4":"4"}`), nil) + err = w.AddDataToPayloadForUT([]byte(`{"4":"4"}`), nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -486,7 +486,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddBinaryVectorToPayload(in, 8) assert.NoError(t, err) - err = w.AddDataToPayload(in2, nil) + err = w.AddDataToPayloadForUT(in2, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -526,7 +526,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddFloatVectorToPayload([]float32{1.0, 2.0}, 1) assert.NoError(t, err) - err = w.AddDataToPayload([]float32{3.0, 4.0}, nil) + err = w.AddDataToPayloadForUT([]float32{3.0, 4.0}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -568,7 +568,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddFloat16VectorToPayload([]byte{1, 2}, 1) assert.NoError(t, err) - err = w.AddDataToPayload([]byte{3, 4}, nil) + err = w.AddDataToPayloadForUT([]byte{3, 4}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -610,7 +610,7 @@ func TestPayload_ReaderAndWriter(t *testing.T) { err = w.AddBFloat16VectorToPayload([]byte{1, 2}, 1) assert.NoError(t, err) - err = w.AddDataToPayload([]byte{3, 4}, nil) + err = w.AddDataToPayloadForUT([]byte{3, 4}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1708,61 +1708,86 @@ func TestPayload_ReaderAndWriter(t *testing.T) { }) t.Run("TestVectorArray", func(t *testing.T) { - w, err := NewPayloadWriter(schemapb.DataType_Array) - require.Nil(t, err) + dim := 4 + w, err := NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(dim), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) require.NotNil(t, w) - err = w.AddOneVectorArrayToPayload(&schemapb.VectorField{ - Data: &schemapb.VectorField_FloatVector{ - FloatVector: &schemapb.FloatArray{ - Data: []float32{1.0, 2.0, 3.0, 4.0}, + // Create VectorArrayFieldData with 3 rows + vectorArrayData := &VectorArrayFieldData{ + Data: []*schemapb.VectorField{ + { + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, + }, + }, + }, + { + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{11.0, 22.0, 33.0, 44.0}, + }, + }, + }, + { + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{111.0, 222.0, 333.0, 444.0, 555.0, 666.0, 777.0, 888.0, 999.0, 1000.0, 1111.0, 1212.0, 1313.0, 1414.0, 1515.0, 1616.0, 1717.0, 1818.0, 1919.0, 2020.0}, + }, + }, }, }, - }) - assert.NoError(t, err) - err = w.AddOneVectorArrayToPayload(&schemapb.VectorField{ - Data: &schemapb.VectorField_FloatVector{ - FloatVector: &schemapb.FloatArray{ - Data: []float32{11.0, 22.0, 33.0, 44.0}, - }, - }, - }) - assert.NoError(t, err) - err = w.AddOneVectorArrayToPayload(&schemapb.VectorField{ - Data: &schemapb.VectorField_FloatVector{ - FloatVector: &schemapb.FloatArray{ - Data: []float32{111.0, 222.0, 333.0, 444.0}, - }, - }, - }) + ElementType: schemapb.DataType_FloatVector, + } + + err = w.AddVectorArrayFieldDataToPayload(vectorArrayData) assert.NoError(t, err) + err = w.FinishPayloadWriter() assert.NoError(t, err) + length, err := w.GetPayloadLengthFromWriter() assert.NoError(t, err) - assert.Equal(t, length, 3) + assert.Equal(t, 3, length) + buffer, err := w.GetPayloadBufferFromWriter() assert.NoError(t, err) r, err := NewPayloadReader(schemapb.DataType_ArrayOfVector, buffer, false) assert.NoError(t, err) + length, err = r.GetPayloadLengthFromReader() assert.NoError(t, err) - assert.Equal(t, length, 3) + assert.Equal(t, 3, length) - arrayList, err := r.GetVectorArrayFromPayload() + vectorArrays, err := r.GetVectorArrayFromPayload() assert.NoError(t, err) - assert.EqualValues(t, []float32{1.0, 2.0, 3.0, 4.0}, arrayList[0].GetFloatVector().GetData()) - assert.EqualValues(t, []float32{11.0, 22.0, 33.0, 44.0}, arrayList[1].GetFloatVector().GetData()) - assert.EqualValues(t, []float32{111.0, 222.0, 333.0, 444.0}, arrayList[2].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, vectorArrays[0].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{11.0, 22.0, 33.0, 44.0}, vectorArrays[1].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{111.0, 222.0, 333.0, 444.0, 555.0, 666.0, 777.0, 888.0, 999.0, 1000.0, 1111.0, 1212.0, 1313.0, 1414.0, 1515.0, 1616.0, 1717.0, 1818.0, 1919.0, 2020.0}, vectorArrays[2].GetFloatVector().GetData()) + for _, v := range vectorArrays { + assert.Equal(t, int64(dim), v.GetDim()) + } iArrayList, _, _, err := r.GetDataFromPayload() - arrayList = iArrayList.([]*schemapb.VectorField) + vectorArrays = iArrayList.([]*schemapb.VectorField) assert.NoError(t, err) - assert.EqualValues(t, []float32{1.0, 2.0, 3.0, 4.0}, arrayList[0].GetFloatVector().GetData()) - assert.EqualValues(t, []float32{11.0, 22.0, 33.0, 44.0}, arrayList[1].GetFloatVector().GetData()) - assert.EqualValues(t, []float32{111.0, 222.0, 333.0, 444.0}, arrayList[2].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, vectorArrays[0].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{11.0, 22.0, 33.0, 44.0}, vectorArrays[1].GetFloatVector().GetData()) + assert.EqualValues(t, []float32{111.0, 222.0, 333.0, 444.0, 555.0, 666.0, 777.0, 888.0, 999.0, 1000.0, 1111.0, 1212.0, 1313.0, 1414.0, 1515.0, 1616.0, 1717.0, 1818.0, 1919.0, 2020.0}, vectorArrays[2].GetFloatVector().GetData()) + for _, v := range vectorArrays { + assert.Equal(t, int64(dim), v.GetDim()) + } + r.ReleasePayloadReader() w.ReleasePayloadWriter() }) @@ -1776,7 +1801,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddBoolToPayload([]bool{true, false, false, false}, []bool{true, false, true, false}) assert.NoError(t, err) - err = w.AddDataToPayload([]bool{true, false, false, false}, []bool{true, false, true, false}) + err = w.AddDataToPayloadForUT([]bool{true, false, false, false}, []bool{true, false, true, false}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1813,7 +1838,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddInt8ToPayload([]int8{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]int8{4, 5, 6}, []bool{true, false, true}) + err = w.AddDataToPayloadForUT([]int8{4, 5, 6}, []bool{true, false, true}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1853,7 +1878,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddInt16ToPayload([]int16{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]int16{1, 2, 3}, []bool{true, false, true}) + err = w.AddDataToPayloadForUT([]int16{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1891,7 +1916,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddInt32ToPayload([]int32{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]int32{1, 2, 3}, []bool{true, false, true}) + err = w.AddDataToPayloadForUT([]int32{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1930,7 +1955,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddInt64ToPayload([]int64{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]int64{1, 2, 3}, []bool{true, false, true}) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, []bool{true, false, true}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -1969,7 +1994,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddFloatToPayload([]float32{1.0, 2.0, 3.0}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]float32{1.0, 2.0, 3.0}, []bool{false, true, false}) + err = w.AddDataToPayloadForUT([]float32{1.0, 2.0, 3.0}, []bool{false, true, false}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -2008,7 +2033,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { err = w.AddDoubleToPayload([]float64{1.0, 2.0, 3.0}, []bool{true, false, true}) assert.NoError(t, err) - err = w.AddDataToPayload([]float64{1.0, 2.0, 3.0}, []bool{false, true, false}) + err = w.AddDataToPayloadForUT([]float64{1.0, 2.0, 3.0}, []bool{false, true, false}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -2051,7 +2076,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { assert.NoError(t, err) err = w.AddOneStringToPayload("hello2", true) assert.NoError(t, err) - err = w.AddDataToPayload("hello3", []bool{false}) + err = w.AddDataToPayloadForUT("hello3", []bool{false}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -2117,7 +2142,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { }, }, true) assert.NoError(t, err) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{7, 8}, @@ -2171,7 +2196,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { assert.NoError(t, err) err = w.AddOneJSONToPayload([]byte(`{"3":"3"}`), true) assert.NoError(t, err) - err = w.AddDataToPayload([]byte(`{"4":"4"}`), []bool{false}) + err = w.AddDataToPayloadForUT([]byte(`{"4":"4"}`), []bool{false}) assert.NoError(t, err) err = w.FinishPayloadWriter() assert.NoError(t, err) @@ -2290,25 +2315,25 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_String, WithNullable(true)) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload("hello0", nil) + err = w.AddDataToPayloadForUT("hello0", nil) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_String, WithNullable(true)) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload("hello0", []bool{false, false}) + err = w.AddDataToPayloadForUT("hello0", []bool{false, false}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_String) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload("hello0", []bool{false}) + err = w.AddDataToPayloadForUT("hello0", []bool{false}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_String) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload("hello0", []bool{true}) + err = w.AddDataToPayloadForUT("hello0", []bool{true}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) }) @@ -2316,7 +2341,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_Array, WithNullable(true)) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{1, 2}, @@ -2329,7 +2354,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{1, 2}, @@ -2341,7 +2366,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { w, err = NewPayloadWriter(schemapb.DataType_Array) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{1, 2}, @@ -2353,7 +2378,7 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { w, err = NewPayloadWriter(schemapb.DataType_Array) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload(&schemapb.ScalarField{ + err = w.AddDataToPayloadForUT(&schemapb.ScalarField{ Data: &schemapb.ScalarField_IntData{ IntData: &schemapb.IntArray{ Data: []int32{1, 2}, @@ -2367,25 +2392,25 @@ func TestPayload_NullableReaderAndWriter(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_JSON, WithNullable(true)) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload([]byte(`{"1":"1"}`), nil) + err = w.AddDataToPayloadForUT([]byte(`{"1":"1"}`), nil) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_JSON, WithNullable(true)) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload([]byte(`{"1":"1"}`), []bool{false, false}) + err = w.AddDataToPayloadForUT([]byte(`{"1":"1"}`), []bool{false, false}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_JSON) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload([]byte(`{"1":"1"}`), []bool{false}) + err = w.AddDataToPayloadForUT([]byte(`{"1":"1"}`), []bool{false}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) w, err = NewPayloadWriter(schemapb.DataType_JSON) require.Nil(t, err) require.NotNil(t, w) - err = w.AddDataToPayload([]byte(`{"1":"1"}`), []bool{true}) + err = w.AddDataToPayloadForUT([]byte(`{"1":"1"}`), []bool{true}) assert.ErrorIs(t, err, merr.ErrParameterInvalid) }) } diff --git a/internal/storage/payload_writer.go b/internal/storage/payload_writer.go index 67eb6f4de0..e5a8b49193 100644 --- a/internal/storage/payload_writer.go +++ b/internal/storage/payload_writer.go @@ -60,8 +60,15 @@ func WithDim(dim int) PayloadWriterOptions { } } +func WithElementType(elementType schemapb.DataType) PayloadWriterOptions { + return func(w *NativePayloadWriter) { + w.elementType = &elementType + } +} + type NativePayloadWriter struct { dataType schemapb.DataType + elementType *schemapb.DataType arrowType arrow.DataType builder array.Builder finished bool @@ -101,12 +108,26 @@ func NewPayloadWriter(colType schemapb.DataType, options ...PayloadWriterOptions } else { w.dim = NewNullableInt(1) } - w.arrowType = MilvusDataTypeToArrowType(colType, *w.dim.Value) - w.builder = array.NewBuilder(memory.DefaultAllocator, w.arrowType) + + // Handle ArrayOfVector type with elementType + if colType == schemapb.DataType_ArrayOfVector { + if w.elementType == nil { + return nil, merr.WrapErrParameterInvalidMsg("ArrayOfVector requires elementType, use WithElementType option") + } + arrowType, err := VectorArrayToArrowType(*w.elementType) + if err != nil { + return nil, err + } + w.arrowType = arrowType + w.builder = array.NewListBuilder(memory.DefaultAllocator, arrowType.(*arrow.ListType).Elem()) + } else { + w.arrowType = MilvusDataTypeToArrowType(colType, *w.dim.Value) + w.builder = array.NewBuilder(memory.DefaultAllocator, w.arrowType) + } return w, nil } -func (w *NativePayloadWriter) AddDataToPayload(data interface{}, validData []bool) error { +func (w *NativePayloadWriter) AddDataToPayloadForUT(data interface{}, validData []bool) error { switch w.dataType { case schemapb.DataType_Bool: val, ok := data.([]bool) @@ -250,11 +271,11 @@ func (w *NativePayloadWriter) AddDataToPayload(data interface{}, validData []boo } return w.AddInt8VectorToPayload(val, w.dim.GetValue()) case schemapb.DataType_ArrayOfVector: - val, ok := data.(*schemapb.VectorField) + val, ok := data.(*VectorArrayFieldData) if !ok { - return merr.WrapErrParameterInvalidMsg("incorrect data type") + return merr.WrapErrParameterInvalidMsg("incorrect data type: expected *VectorArrayFieldData") } - return w.AddOneVectorArrayToPayload(val) + return w.AddVectorArrayFieldDataToPayload(val) default: return errors.New("unsupported datatype") } @@ -742,26 +763,6 @@ func (w *NativePayloadWriter) AddInt8VectorToPayload(data []int8, dim int) error return nil } -func (w *NativePayloadWriter) AddOneVectorArrayToPayload(data *schemapb.VectorField) error { - if w.finished { - return errors.New("can't append data to finished vector array payload") - } - - bytes, err := proto.Marshal(data) - if err != nil { - return errors.New("Marshal VectorField failed") - } - - builder, ok := w.builder.(*array.BinaryBuilder) - if !ok { - return errors.New("failed to cast VectorArrayBuilder") - } - - builder.Append(bytes) - - return nil -} - func (w *NativePayloadWriter) FinishPayloadWriter() error { if w.finished { return errors.New("can't reuse a finished writer") @@ -769,10 +770,24 @@ func (w *NativePayloadWriter) FinishPayloadWriter() error { w.finished = true + // Prepare metadata for VectorArray type + var metadata arrow.Metadata + if w.dataType == schemapb.DataType_ArrayOfVector { + if w.elementType == nil { + return errors.New("element type for DataType_ArrayOfVector must be set") + } + + metadata = arrow.NewMetadata( + []string{"elementType", "dim"}, + []string{fmt.Sprintf("%d", int32(*w.elementType)), fmt.Sprintf("%d", w.dim.GetValue())}, + ) + } + field := arrow.Field{ Name: "val", Type: w.arrowType, Nullable: w.nullable, + Metadata: metadata, } schema := arrow.NewSchema([]arrow.Field{ field, @@ -787,11 +802,19 @@ func (w *NativePayloadWriter) FinishPayloadWriter() error { table := array.NewTable(schema, []arrow.Column{column}, int64(column.Len())) defer table.Release() + arrowWriterProps := pqarrow.DefaultWriterProps() + if w.dataType == schemapb.DataType_ArrayOfVector { + // Store metadata in the Arrow writer properties + arrowWriterProps = pqarrow.NewArrowWriterProperties( + pqarrow.WithStoreSchema(), + ) + } + return pqarrow.WriteTable(table, w.output, 1024*1024*1024, w.writerProps, - pqarrow.DefaultWriterProps(), + arrowWriterProps, ) } @@ -869,8 +892,84 @@ func MilvusDataTypeToArrowType(dataType schemapb.DataType, dim int) arrow.DataTy ByteWidth: dim, } case schemapb.DataType_ArrayOfVector: - return &arrow.BinaryType{} + // ArrayOfVector requires elementType, should use VectorArrayToArrowType instead + panic("ArrayOfVector type requires elementType information, use VectorArrayToArrowType") default: panic("unsupported data type") } } + +// VectorArrayToArrowType converts VectorArray type with elementType to Arrow ListArray type +func VectorArrayToArrowType(elementType schemapb.DataType) (arrow.DataType, error) { + var childType arrow.DataType + + switch elementType { + case schemapb.DataType_FloatVector: + childType = arrow.PrimitiveTypes.Float32 + case schemapb.DataType_BinaryVector: + return nil, merr.WrapErrParameterInvalidMsg("BinaryVector in VectorArray not implemented yet") + case schemapb.DataType_Float16Vector: + return nil, merr.WrapErrParameterInvalidMsg("Float16Vector in VectorArray not implemented yet") + case schemapb.DataType_BFloat16Vector: + return nil, merr.WrapErrParameterInvalidMsg("BFloat16Vector in VectorArray not implemented yet") + case schemapb.DataType_Int8Vector: + return nil, merr.WrapErrParameterInvalidMsg("Int8Vector in VectorArray not implemented yet") + default: + return nil, merr.WrapErrParameterInvalidMsg(fmt.Sprintf("unsupported element type in VectorArray: %s", elementType.String())) + } + + return arrow.ListOf(childType), nil +} + +// AddVectorArrayFieldDataToPayload adds VectorArrayFieldData to payload using Arrow ListArray +func (w *NativePayloadWriter) AddVectorArrayFieldDataToPayload(data *VectorArrayFieldData) error { + if w.finished { + return errors.New("can't append data to finished vector array payload") + } + + if len(data.Data) == 0 { + return errors.New("can't add empty vector array field data") + } + + builder, ok := w.builder.(*array.ListBuilder) + if !ok { + return errors.New("failed to cast to ListBuilder for VectorArray") + } + + switch data.ElementType { + case schemapb.DataType_FloatVector: + return w.addFloatVectorArrayToPayload(builder, data) + case schemapb.DataType_BinaryVector: + return merr.WrapErrParameterInvalidMsg("BinaryVector in VectorArray not implemented yet") + case schemapb.DataType_Float16Vector: + return merr.WrapErrParameterInvalidMsg("Float16Vector in VectorArray not implemented yet") + case schemapb.DataType_BFloat16Vector: + return merr.WrapErrParameterInvalidMsg("BFloat16Vector in VectorArray not implemented yet") + case schemapb.DataType_Int8Vector: + return merr.WrapErrParameterInvalidMsg("Int8Vector in VectorArray not implemented yet") + default: + return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("unsupported element type in VectorArray: %s", data.ElementType.String())) + } +} + +// addFloatVectorArrayToPayload handles FloatVector elements in VectorArray +func (w *NativePayloadWriter) addFloatVectorArrayToPayload(builder *array.ListBuilder, data *VectorArrayFieldData) error { + valueBuilder := builder.ValueBuilder().(*array.Float32Builder) + + for _, vectorField := range data.Data { + if vectorField.GetFloatVector() == nil { + return merr.WrapErrParameterInvalidMsg("expected FloatVector but got different type") + } + + builder.Append(true) + + floatData := vectorField.GetFloatVector().GetData() + if len(floatData) == 0 { + return merr.WrapErrParameterInvalidMsg("empty vector data not allowed") + } + + valueBuilder.AppendValues(floatData, nil) + } + + return nil +} diff --git a/internal/storage/payload_writer_test.go b/internal/storage/payload_writer_test.go index 46b7f01de1..fb7561b736 100644 --- a/internal/storage/payload_writer_test.go +++ b/internal/storage/payload_writer_test.go @@ -297,29 +297,193 @@ func TestPayloadWriter_Failed(t *testing.T) { err = w.AddFloatToPayload(data, nil) require.Error(t, err) }) +} - t.Run("Test ArrayOfFloatVector", func(t *testing.T) { - // The dim is not used for ArrayOfVector type for payload writer as each row contains this info - w, err := NewPayloadWriter(schemapb.DataType_ArrayOfVector, WithDim(1)) - require.Nil(t, err) +func TestPayloadWriter_ArrayOfVector(t *testing.T) { + t.Run("Test ArrayOfFloatVector - Basic", func(t *testing.T) { + dim := 128 + numRows := 100 + vectorsPerRow := 5 + + // Create test data + vectorArrayData := &VectorArrayFieldData{ + Data: make([]*schemapb.VectorField, numRows), + ElementType: schemapb.DataType_FloatVector, + } + + for i := 0; i < numRows; i++ { + floatData := make([]float32, vectorsPerRow*dim) + for j := 0; j < len(floatData); j++ { + floatData[j] = float32(i*1000 + j) // Predictable values for verification + } + + vectorArrayData.Data[i] = &schemapb.VectorField{ + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: floatData, + }, + }, + } + } + + w, err := NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(dim), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) require.NotNil(t, w) + err = w.AddVectorArrayFieldDataToPayload(vectorArrayData) + require.NoError(t, err) + err = w.FinishPayloadWriter() require.NoError(t, err) - err = w.AddOneVectorArrayToPayload(&schemapb.VectorField{ - Dim: 8, - }) - require.Error(t, err) + // Verify results + buffer, err := w.GetPayloadBufferFromWriter() + require.NoError(t, err) + require.NotEmpty(t, buffer) - w, err = NewPayloadWriter(schemapb.DataType_Int64) - require.Nil(t, err) + length, err := w.GetPayloadLengthFromWriter() + require.NoError(t, err) + require.Equal(t, numRows, length) + }) + + t.Run("Test ArrayOfFloatVector - Error Cases", func(t *testing.T) { + // Test missing ElementType + _, err := NewPayloadWriter(schemapb.DataType_ArrayOfVector, WithDim(128)) + require.Error(t, err) + require.Contains(t, err.Error(), "requires elementType") + + // Test with correct setup + w, err := NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(128), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) require.NotNil(t, w) - err = w.AddOneVectorArrayToPayload(&schemapb.VectorField{ - Dim: 8, - }) + // Test adding empty data + emptyData := &VectorArrayFieldData{ + Data: []*schemapb.VectorField{}, + ElementType: schemapb.DataType_FloatVector, + } + err = w.AddVectorArrayFieldDataToPayload(emptyData) require.Error(t, err) + require.Contains(t, err.Error(), "empty vector array") + + // Test incorrect data type with AddDataToPayloadForUT + w, err = NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(128), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) + + wrongData := "not a VectorArrayFieldData" + err = w.AddDataToPayloadForUT(wrongData, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "incorrect data type") + }) + + t.Run("Test ArrayOfFloatVector - Multiple Batches", func(t *testing.T) { + // Test adding multiple batches of vector arrays + dim := 64 + batchSize := 50 + numBatches := 3 + vectorsPerRow := 3 + + w, err := NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(dim), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) + + totalRows := 0 + for batch := 0; batch < numBatches; batch++ { + batchData := &VectorArrayFieldData{ + Data: make([]*schemapb.VectorField, batchSize), + ElementType: schemapb.DataType_FloatVector, + } + + for i := 0; i < batchSize; i++ { + floatData := make([]float32, vectorsPerRow*dim) + for j := 0; j < len(floatData); j++ { + floatData[j] = float32(batch*10000 + i*100 + j) + } + + batchData.Data[i] = &schemapb.VectorField{ + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: floatData, + }, + }, + } + } + + err = w.AddVectorArrayFieldDataToPayload(batchData) + require.NoError(t, err) + totalRows += batchSize + } + + err = w.FinishPayloadWriter() + require.NoError(t, err) + + length, err := w.GetPayloadLengthFromWriter() + require.NoError(t, err) + require.Equal(t, totalRows, length) + }) + + t.Run("Test ArrayOfFloatVector - Variable Vectors Per Row", func(t *testing.T) { + // Test with different number of vectors per row + dim := 32 + numRows := 20 + + w, err := NewPayloadWriter( + schemapb.DataType_ArrayOfVector, + WithDim(dim), + WithElementType(schemapb.DataType_FloatVector), + ) + require.NoError(t, err) + + vectorArrayData := &VectorArrayFieldData{ + Data: make([]*schemapb.VectorField, numRows), + ElementType: schemapb.DataType_FloatVector, + } + + for i := 0; i < numRows; i++ { + // Variable number of vectors per row (1 to 10) + vectorsPerRow := (i % 10) + 1 + floatData := make([]float32, vectorsPerRow*dim) + + for j := 0; j < len(floatData); j++ { + floatData[j] = float32(i*100 + j) + } + + vectorArrayData.Data[i] = &schemapb.VectorField{ + Dim: int64(dim), + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: floatData, + }, + }, + } + } + + err = w.AddVectorArrayFieldDataToPayload(vectorArrayData) + require.NoError(t, err) + + err = w.FinishPayloadWriter() + require.NoError(t, err) + + length, err := w.GetPayloadLengthFromWriter() + require.NoError(t, err) + require.Equal(t, numRows, length) }) } @@ -330,7 +494,7 @@ func TestParquetEncoding(t *testing.T) { w, err := NewPayloadWriter(schemapb.DataType_Int64, WithWriterProps(getFieldWriterProps(field))) assert.NoError(t, err) - err = w.AddDataToPayload([]int64{1, 2, 3}, nil) + err = w.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) err = w.FinishPayloadWriter() diff --git a/internal/storage/print_binlog_test.go b/internal/storage/print_binlog_test.go index 22f1b2aaf6..cfe464c7d0 100644 --- a/internal/storage/print_binlog_test.go +++ b/internal/storage/print_binlog_test.go @@ -40,21 +40,21 @@ func TestPrintBinlogFilesInt64(t *testing.T) { e1, err := w.NextInsertEventWriter() assert.NoError(t, err) - err = e1.AddDataToPayload([]int64{1, 2, 3}, nil) + err = e1.AddDataToPayloadForUT([]int64{1, 2, 3}, nil) assert.NoError(t, err) - err = e1.AddDataToPayload([]int32{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int32{4, 5, 6}, nil) assert.Error(t, err) - err = e1.AddDataToPayload([]int64{4, 5, 6}, nil) + err = e1.AddDataToPayloadForUT([]int64{4, 5, 6}, nil) assert.NoError(t, err) e1.SetEventTimestamp(tsoutil.ComposeTS(curTS+10*60*1000, 0), tsoutil.ComposeTS(curTS+20*60*1000, 0)) e2, err := w.NextInsertEventWriter() assert.NoError(t, err) - err = e2.AddDataToPayload([]int64{7, 8, 9}, nil) + err = e2.AddDataToPayloadForUT([]int64{7, 8, 9}, nil) assert.NoError(t, err) - err = e2.AddDataToPayload([]bool{true, false, true}, nil) + err = e2.AddDataToPayloadForUT([]bool{true, false, true}, nil) assert.Error(t, err) - err = e2.AddDataToPayload([]int64{10, 11, 12}, nil) + err = e2.AddDataToPayloadForUT([]int64{10, 11, 12}, nil) assert.NoError(t, err) e2.SetEventTimestamp(tsoutil.ComposeTS(curTS+30*60*1000, 0), tsoutil.ComposeTS(curTS+40*60*1000, 0)) diff --git a/internal/storage/schema.go b/internal/storage/schema.go index 788f76062b..afbc871625 100644 --- a/internal/storage/schema.go +++ b/internal/storage/schema.go @@ -30,7 +30,24 @@ func ConvertToArrowSchema(schema *schemapb.CollectionSchema) (*arrow.Schema, err default: dim = 0 } - arrowFields = append(arrowFields, ConvertToArrowField(field, serdeMap[field.DataType].arrowType(dim))) + + elementType := schemapb.DataType_None + if field.DataType == schemapb.DataType_ArrayOfVector { + elementType = field.GetElementType() + } + + arrowType := serdeMap[field.DataType].arrowType(dim, elementType) + arrowField := ConvertToArrowField(field, arrowType) + + // Add extra metadata for ArrayOfVector + if field.DataType == schemapb.DataType_ArrayOfVector { + arrowField.Metadata = arrow.NewMetadata( + []string{packed.ArrowFieldIdMetadataKey, "elementType", "dim"}, + []string{strconv.Itoa(int(field.GetFieldID())), strconv.Itoa(int(elementType)), strconv.Itoa(dim)}, + ) + } + + arrowFields = append(arrowFields, arrowField) return nil } for _, field := range schema.GetFields() { diff --git a/internal/storage/serde.go b/internal/storage/serde.go index ae3ea17623..3157fd0228 100644 --- a/internal/storage/serde.go +++ b/internal/storage/serde.go @@ -92,24 +92,27 @@ func (r *compositeRecord) Retain() { } type serdeEntry struct { - // arrowType returns the arrow type for the given dimension - arrowType func(int) arrow.DataType + // arrowType returns the arrow type for the given dimension and element type + // elementType is only used for ArrayOfVector + arrowType func(dim int, elementType schemapb.DataType) arrow.DataType // deserialize deserializes the i-th element in the array, returns the value and ok. // null is deserialized to nil without checking the type nullability. // if shouldCopy is true, the returned value is copied rather than referenced from arrow array. - deserialize func(arrow.Array, int, bool) (any, bool) + // elementType is only used for ArrayOfVector + deserialize func(a arrow.Array, i int, elementType schemapb.DataType, dim int, shouldCopy bool) (any, bool) // serialize serializes the value to the builder, returns ok. // nil is serialized to null without checking the type nullability. - serialize func(array.Builder, any) bool + // elementType is only used for ArrayOfVector + serialize func(b array.Builder, v any, elementType schemapb.DataType) bool } var serdeMap = func() map[schemapb.DataType]serdeEntry { m := make(map[schemapb.DataType]serdeEntry) m[schemapb.DataType_Bool] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.FixedWidthTypes.Boolean }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -118,7 +121,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -133,10 +136,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Int8] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Int8 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -145,7 +148,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -160,10 +163,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Int16] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Int16 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -172,7 +175,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -187,10 +190,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Int32] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Int32 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -199,7 +202,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -214,10 +217,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Int64] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Int64 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -226,7 +229,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -241,10 +244,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Float] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Float32 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -253,7 +256,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -268,10 +271,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Double] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Float64 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -280,7 +283,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -295,10 +298,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_Timestamptz] = serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.PrimitiveTypes.Int64 }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, _ int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -307,7 +310,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -322,10 +325,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } stringEntry := serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.BinaryTypes.String }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -338,7 +341,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -360,10 +363,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { // We're not using the deserialized data in go, so we can skip the heavy pb serde. // If there is need in the future, just assign it to m[schemapb.DataType_Array] eagerArrayEntry := serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.BinaryTypes.Binary }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -375,7 +378,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -394,10 +397,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { _ = eagerArrayEntry byteEntry := serdeEntry{ - arrowType: func(i int) arrow.DataType { + arrowType: func(_ int, _ schemapb.DataType) arrow.DataType { return arrow.BinaryTypes.Binary }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, dim int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -412,7 +415,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -441,9 +444,59 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { m[schemapb.DataType_Array] = eagerArrayEntry m[schemapb.DataType_JSON] = byteEntry - m[schemapb.DataType_ArrayOfVector] = byteEntry - fixedSizeDeserializer := func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + // ArrayOfVector now implements the standard interface with elementType parameter + m[schemapb.DataType_ArrayOfVector] = serdeEntry{ + arrowType: func(_ int, elementType schemapb.DataType) arrow.DataType { + return getArrayOfVectorArrowType(elementType) + }, + deserialize: func(a arrow.Array, i int, elementType schemapb.DataType, dim int, shouldCopy bool) (any, bool) { + return deserializeArrayOfVector(a, i, elementType, int64(dim), shouldCopy) + }, + serialize: func(b array.Builder, v any, elementType schemapb.DataType) bool { + vf, ok := v.(*schemapb.VectorField) + if !ok { + return false + } + + if vf == nil { + b.AppendNull() + return true + } + + builder, ok := b.(*array.ListBuilder) + if !ok { + return false + } + + builder.Append(true) + + switch elementType { + case schemapb.DataType_FloatVector: + if vf.GetFloatVector() == nil { + return false + } + valueBuilder := builder.ValueBuilder().(*array.Float32Builder) + valueBuilder.AppendValues(vf.GetFloatVector().GetData(), nil) + return true + + case schemapb.DataType_BinaryVector: + panic("BinaryVector in VectorArray not implemented yet") + case schemapb.DataType_Float16Vector: + panic("Float16Vector in VectorArray not implemented yet") + case schemapb.DataType_BFloat16Vector: + panic("BFloat16Vector in VectorArray not implemented yet") + case schemapb.DataType_Int8Vector: + panic("Int8Vector in VectorArray not implemented yet") + case schemapb.DataType_SparseFloatVector: + panic("SparseFloatVector in VectorArray not implemented yet") + default: + return false + } + }, + } + + fixedSizeDeserializer := func(a arrow.Array, i int, _ schemapb.DataType, _ int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -458,7 +511,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false } - fixedSizeSerializer := func(b array.Builder, v any) bool { + fixedSizeSerializer := func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -473,31 +526,31 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } m[schemapb.DataType_BinaryVector] = serdeEntry{ - arrowType: func(i int) arrow.DataType { - return &arrow.FixedSizeBinaryType{ByteWidth: (i + 7) / 8} + arrowType: func(dim int, _ schemapb.DataType) arrow.DataType { + return &arrow.FixedSizeBinaryType{ByteWidth: (dim + 7) / 8} }, deserialize: fixedSizeDeserializer, serialize: fixedSizeSerializer, } m[schemapb.DataType_Float16Vector] = serdeEntry{ - arrowType: func(i int) arrow.DataType { - return &arrow.FixedSizeBinaryType{ByteWidth: i * 2} + arrowType: func(dim int, _ schemapb.DataType) arrow.DataType { + return &arrow.FixedSizeBinaryType{ByteWidth: dim * 2} }, deserialize: fixedSizeDeserializer, serialize: fixedSizeSerializer, } m[schemapb.DataType_BFloat16Vector] = serdeEntry{ - arrowType: func(i int) arrow.DataType { - return &arrow.FixedSizeBinaryType{ByteWidth: i * 2} + arrowType: func(dim int, _ schemapb.DataType) arrow.DataType { + return &arrow.FixedSizeBinaryType{ByteWidth: dim * 2} }, deserialize: fixedSizeDeserializer, serialize: fixedSizeSerializer, } m[schemapb.DataType_Int8Vector] = serdeEntry{ - arrowType: func(i int) arrow.DataType { - return &arrow.FixedSizeBinaryType{ByteWidth: i} + arrowType: func(dim int, _ schemapb.DataType) arrow.DataType { + return &arrow.FixedSizeBinaryType{ByteWidth: dim} }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, _ int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -512,7 +565,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -530,10 +583,10 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { }, } m[schemapb.DataType_FloatVector] = serdeEntry{ - arrowType: func(i int) arrow.DataType { - return &arrow.FixedSizeBinaryType{ByteWidth: i * 4} + arrowType: func(dim int, _ schemapb.DataType) arrow.DataType { + return &arrow.FixedSizeBinaryType{ByteWidth: dim * 4} }, - deserialize: func(a arrow.Array, i int, shouldCopy bool) (any, bool) { + deserialize: func(a arrow.Array, i int, _ schemapb.DataType, _ int, shouldCopy bool) (any, bool) { if a.IsNull(i) { return nil, true } @@ -548,7 +601,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry { } return nil, false }, - serialize: func(b array.Builder, v any) bool { + serialize: func(b array.Builder, v any, _ schemapb.DataType) bool { if v == nil { b.AppendNull() return true @@ -751,6 +804,98 @@ func (sfw *singleFieldRecordWriter) Close() error { return sfw.fw.Close() } +// getArrayOfVectorArrowType returns the appropriate Arrow type for ArrayOfVector based on element type +func getArrayOfVectorArrowType(elementType schemapb.DataType) arrow.DataType { + switch elementType { + case schemapb.DataType_FloatVector: + return arrow.ListOf(arrow.PrimitiveTypes.Float32) + case schemapb.DataType_BinaryVector: + return arrow.ListOf(arrow.PrimitiveTypes.Uint8) + case schemapb.DataType_Float16Vector: + return arrow.ListOf(arrow.PrimitiveTypes.Uint8) + case schemapb.DataType_BFloat16Vector: + return arrow.ListOf(arrow.PrimitiveTypes.Uint8) + case schemapb.DataType_Int8Vector: + return arrow.ListOf(arrow.PrimitiveTypes.Int8) + case schemapb.DataType_SparseFloatVector: + return arrow.ListOf(arrow.BinaryTypes.Binary) + default: + panic(fmt.Sprintf("unsupported element type for ArrayOfVector: %s", elementType.String())) + } +} + +// deserializeArrayOfVector deserializes ArrayOfVector data with known element type +func deserializeArrayOfVector(a arrow.Array, i int, elementType schemapb.DataType, dim int64, shouldCopy bool) (any, bool) { + if a.IsNull(i) { + return nil, true + } + + arr, ok := a.(*array.List) + if !ok || i >= arr.Len() { + return nil, false + } + + start, end := arr.ValueOffsets(i) + totalElements := end - start + if totalElements == 0 { + return nil, false + } + + // Validate dimension for vector types that have fixed dimensions + if dim > 0 && totalElements%dim != 0 { + // Dimension mismatch - data corruption or schema inconsistency + return nil, false + } + + valuesArray := arr.ListValues() + + switch elementType { + case schemapb.DataType_FloatVector: + floatArray, ok := valuesArray.(*array.Float32) + if !ok { + return nil, false + } + + // Handle data copying based on shouldCopy parameter + var floatData []float32 + if shouldCopy { + // Explicitly requested copy + floatData = make([]float32, totalElements) + for j := start; j < end; j++ { + floatData[j-start] = floatArray.Value(int(j)) + } + } else { + // Try to avoid copying - use a slice of the underlying data + // This creates a slice that references the same underlying array + allData := floatArray.Float32Values() + floatData = allData[start:end] + } + + vectorField := &schemapb.VectorField{ + Dim: dim, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: floatData, + }, + }, + } + return vectorField, true + + case schemapb.DataType_BinaryVector: + panic("BinaryVector in VectorArray deserialization not implemented yet") + case schemapb.DataType_Float16Vector: + panic("Float16Vector in VectorArray deserialization not implemented yet") + case schemapb.DataType_BFloat16Vector: + panic("BFloat16Vector in VectorArray deserialization not implemented yet") + case schemapb.DataType_Int8Vector: + panic("Int8Vector in VectorArray deserialization not implemented yet") + case schemapb.DataType_SparseFloatVector: + panic("SparseFloatVector in VectorArray deserialization not implemented yet") + default: + panic(fmt.Sprintf("unsupported element type for ArrayOfVector deserialization: %s", elementType.String())) + } +} + func newSingleFieldRecordWriter(field *schemapb.FieldSchema, writer io.Writer, opts ...RecordWriterOptions) (*singleFieldRecordWriter, error) { // calculate memory expansion ratio // arrays are serialized by protobuf, where int values may be compacted, see https://protobuf.dev/reference/go/size @@ -769,13 +914,28 @@ func newSingleFieldRecordWriter(field *schemapb.FieldSchema, writer io.Writer, o return 1 } dim, _ := typeutil.GetDim(field) + + var fieldMetadata arrow.Metadata + var arrowType arrow.DataType + elementType := schemapb.DataType_None + + if field.DataType == schemapb.DataType_ArrayOfVector { + elementType = field.GetElementType() + fieldMetadata = arrow.NewMetadata( + []string{"elementType", "dim"}, + []string{fmt.Sprintf("%d", int32(elementType)), fmt.Sprintf("%d", dim)}, + ) + } + arrowType = serdeMap[field.DataType].arrowType(int(dim), elementType) + w := &singleFieldRecordWriter{ fieldId: field.FieldID, schema: arrow.NewSchema([]arrow.Field{ { Name: strconv.Itoa(int(field.FieldID)), - Type: serdeMap[field.DataType].arrowType(int(dim)), + Type: arrowType, Nullable: true, // No nullable check here. + Metadata: fieldMetadata, }, }, nil), writerProps: parquet.NewWriterProperties( @@ -787,7 +947,17 @@ func newSingleFieldRecordWriter(field *schemapb.FieldSchema, writer io.Writer, o for _, o := range opts { o(w) } - fw, err := pqarrow.NewFileWriter(w.schema, writer, w.writerProps, pqarrow.DefaultWriterProps()) + + // Use appropriate Arrow writer properties for ArrayOfVector + arrowWriterProps := pqarrow.DefaultWriterProps() + if field.DataType == schemapb.DataType_ArrayOfVector { + // Ensure schema metadata is preserved for ArrayOfVector + arrowWriterProps = pqarrow.NewArrowWriterProperties( + pqarrow.WithStoreSchema(), + ) + } + + fw, err := pqarrow.NewFileWriter(w.schema, writer, w.writerProps, arrowWriterProps) if err != nil { return nil, err } @@ -962,8 +1132,14 @@ func BuildRecord(b *array.RecordBuilder, data *InsertData, schema *schemapb.Coll return merr.WrapErrServiceInternal(fmt.Sprintf("row num is 0 for field %s", field.Name)) } + // Get element type for ArrayOfVector, otherwise use None + elementType := schemapb.DataType_None + if field.DataType == schemapb.DataType_ArrayOfVector { + elementType = field.GetElementType() + } + for j := 0; j < fieldData.RowNum(); j++ { - ok = typeEntry.serialize(fBuilder, fieldData.GetRow(j)) + ok = typeEntry.serialize(fBuilder, fieldData.GetRow(j), elementType) if !ok { return merr.WrapErrServiceInternal(fmt.Sprintf("serialize error on type %s", field.DataType.String())) } diff --git a/internal/storage/serde_events.go b/internal/storage/serde_events.go index 510df8344d..78b58a0071 100644 --- a/internal/storage/serde_events.go +++ b/internal/storage/serde_events.go @@ -320,7 +320,18 @@ func valueDeserializer(r Record, v []*Value, fields []*schemapb.FieldSchema, sho m[j] = nil } } else { - d, ok := serdeMap[dt].deserialize(r.Column(j), i, shouldCopy) + // Get element type and dim for ArrayOfVector, otherwise use defaults + elementType := schemapb.DataType_None + dim := 0 + if typeutil.IsVectorType(dt) { + dimValue, _ := typeutil.GetDim(f) + dim = int(dimValue) + } + if dt == schemapb.DataType_ArrayOfVector { + elementType = f.GetElementType() + } + + d, ok := serdeMap[dt].deserialize(r.Column(j), i, elementType, dim, shouldCopy) if ok { m[j] = d // TODO: avoid memory copy here. } else { @@ -577,9 +588,18 @@ func ValueSerializer(v []*Value, schema *schemapb.CollectionSchema) (Record, err builders := make(map[FieldID]array.Builder, len(allFieldsSchema)) types := make(map[FieldID]schemapb.DataType, len(allFieldsSchema)) + elementTypes := make(map[FieldID]schemapb.DataType, len(allFieldsSchema)) // For ArrayOfVector for _, f := range allFieldsSchema { dim, _ := typeutil.GetDim(f) - builders[f.FieldID] = array.NewBuilder(memory.DefaultAllocator, serdeMap[f.DataType].arrowType(int(dim))) + + elementType := schemapb.DataType_None + if f.DataType == schemapb.DataType_ArrayOfVector { + elementType = f.GetElementType() + elementTypes[f.FieldID] = elementType + } + + arrowType := serdeMap[f.DataType].arrowType(int(dim), elementType) + builders[f.FieldID] = array.NewBuilder(memory.DefaultAllocator, arrowType) builders[f.FieldID].Reserve(len(v)) // reserve space to avoid copy types[f.FieldID] = f.DataType } @@ -592,7 +612,14 @@ func ValueSerializer(v []*Value, schema *schemapb.CollectionSchema) (Record, err if !ok { panic("unknown type") } - ok = typeEntry.serialize(builders[fid], e) + + // Get element type for ArrayOfVector, otherwise use None + elementType := schemapb.DataType_None + if types[fid] == schemapb.DataType_ArrayOfVector { + elementType = elementTypes[fid] + } + + ok = typeEntry.serialize(builders[fid], e, elementType) if !ok { return nil, merr.WrapErrServiceInternal(fmt.Sprintf("serialize error on type %s", types[fid])) } @@ -1252,7 +1279,7 @@ func (dsw *MultiFieldDeltalogStreamWriter) GetRecordWriter() (RecordWriter, erro fields := []arrow.Field{ { Name: "pk", - Type: serdeMap[dsw.pkType].arrowType(0), + Type: serdeMap[dsw.pkType].arrowType(0, schemapb.DataType_None), Nullable: false, }, { @@ -1329,7 +1356,7 @@ func newDeltalogMultiFieldWriter(eventWriter *MultiFieldDeltalogStreamWriter, ba fields := []arrow.Field{ { Name: "pk", - Type: serdeMap[schemapb.DataType(v[0].PkType)].arrowType(0), + Type: serdeMap[schemapb.DataType(v[0].PkType)].arrowType(0, schemapb.DataType_None), Nullable: false, }, { diff --git a/internal/storage/serde_test.go b/internal/storage/serde_test.go index 817bea5d24..98f4dcf658 100644 --- a/internal/storage/serde_test.go +++ b/internal/storage/serde_test.go @@ -27,6 +27,7 @@ import ( "github.com/apache/arrow/go/v17/arrow/memory" "github.com/stretchr/testify/assert" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" ) @@ -101,17 +102,16 @@ func TestSerDe(t *testing.T) { {"test bfloat16 vector null", args{dt: schemapb.DataType_BFloat16Vector, v: nil}, nil, true}, {"test bfloat16 vector negative", args{dt: schemapb.DataType_BFloat16Vector, v: -1}, nil, false}, {"test int8 vector", args{dt: schemapb.DataType_Int8Vector, v: []int8{10}}, []int8{10}, true}, - {"test array of vector", args{dt: schemapb.DataType_ArrayOfVector, v: "{}"}, nil, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { dt := tt.args.dt v := tt.args.v - builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[dt].arrowType(1)) - serdeMap[dt].serialize(builder, v) + builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[dt].arrowType(1, schemapb.DataType_None)) + serdeMap[dt].serialize(builder, v, schemapb.DataType_None) // assert.True(t, ok) a := builder.NewArray() - got, got1 := serdeMap[dt].deserialize(a, 0, false) + got, got1 := serdeMap[dt].deserialize(a, 0, schemapb.DataType_None, 0, false) if !reflect.DeepEqual(got, tt.want) { t.Errorf("deserialize() got = %v, want %v", got, tt.want) } @@ -140,20 +140,20 @@ func TestSerDeCopy(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dt := tt.dt v := tt.v - builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[dt].arrowType(1)) + builder := array.NewBuilder(memory.DefaultAllocator, serdeMap[dt].arrowType(1, schemapb.DataType_None)) defer builder.Release() - serdeMap[dt].serialize(builder, v) + serdeMap[dt].serialize(builder, v, schemapb.DataType_None) a := builder.NewArray() // Test deserialize with shouldCopy parameter - copy, got1 := serdeMap[dt].deserialize(a, 0, true) + copy, got1 := serdeMap[dt].deserialize(a, 0, schemapb.DataType_None, 0, true) if !got1 { t.Errorf("deserialize() failed for %s", tt.name) } if !reflect.DeepEqual(copy, tt.v) { t.Errorf("deserialize() got = %v, want %v", copy, tt.v) } - ref, _ := serdeMap[dt].deserialize(a, 0, false) + ref, _ := serdeMap[dt].deserialize(a, 0, schemapb.DataType_None, 0, false) // check the unsafe pointers of copy and ref are different switch v := copy.(type) { case []byte: @@ -266,3 +266,192 @@ func TestCalculateArraySize(t *testing.T) { }) } } + +func TestArrayOfVectorArrowType(t *testing.T) { + tests := []struct { + name string + elementType schemapb.DataType + expectedChild arrow.DataType + }{ + { + name: "FloatVector", + elementType: schemapb.DataType_FloatVector, + expectedChild: arrow.PrimitiveTypes.Float32, + }, + { + name: "BinaryVector", + elementType: schemapb.DataType_BinaryVector, + expectedChild: arrow.PrimitiveTypes.Uint8, + }, + { + name: "Float16Vector", + elementType: schemapb.DataType_Float16Vector, + expectedChild: arrow.PrimitiveTypes.Uint8, + }, + { + name: "BFloat16Vector", + elementType: schemapb.DataType_BFloat16Vector, + expectedChild: arrow.PrimitiveTypes.Uint8, + }, + { + name: "Int8Vector", + elementType: schemapb.DataType_Int8Vector, + expectedChild: arrow.PrimitiveTypes.Int8, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + arrowType := getArrayOfVectorArrowType(tt.elementType) + assert.NotNil(t, arrowType) + + listType, ok := arrowType.(*arrow.ListType) + assert.True(t, ok) + assert.Equal(t, tt.expectedChild, listType.Elem()) + }) + } +} + +func TestArrayOfVectorSerialization(t *testing.T) { + tests := []struct { + name string + elementType schemapb.DataType + dim int + vectors []*schemapb.VectorField + }{ + { + name: "FloatVector array", + elementType: schemapb.DataType_FloatVector, + dim: 4, + vectors: []*schemapb.VectorField{ + { + Dim: 4, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{1.0, 2.0, 3.0, 4.0}, + }, + }, + }, + { + Dim: 4, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entry := serdeMap[schemapb.DataType_ArrayOfVector] + + arrowType := entry.arrowType(tt.dim, tt.elementType) + assert.NotNil(t, arrowType) + + builder := array.NewBuilder(memory.DefaultAllocator, arrowType) + defer builder.Release() + + for _, vector := range tt.vectors { + ok := entry.serialize(builder, vector, tt.elementType) + assert.True(t, ok) + } + + arr := builder.NewArray() + defer arr.Release() + + for i, expectedVector := range tt.vectors { + result, ok := entry.deserialize(arr, i, tt.elementType, tt.dim, false) + assert.True(t, ok) + + if expectedVector == nil { + assert.Nil(t, result) + } else { + resultVector, ok := result.(*schemapb.VectorField) + assert.True(t, ok) + assert.NotNil(t, resultVector) + + assert.Equal(t, expectedVector.GetDim(), resultVector.GetDim()) + + if tt.elementType == schemapb.DataType_FloatVector { + expectedData := expectedVector.GetFloatVector().GetData() + resultData := resultVector.GetFloatVector().GetData() + assert.Equal(t, expectedData, resultData) + } + } + } + }) + } +} + +func TestArrayOfVectorIntegration(t *testing.T) { + // Test the full integration with BuildRecord + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + FieldID: 100, + Name: "vec_array", + DataType: schemapb.DataType_ArrayOfVector, + ElementType: schemapb.DataType_FloatVector, + TypeParams: []*commonpb.KeyValuePair{ + {Key: "dim", Value: "4"}, + }, + }, + }, + } + + // Create insert data + insertData := &InsertData{ + Data: map[FieldID]FieldData{ + 100: &VectorArrayFieldData{ + Data: []*schemapb.VectorField{ + { + Dim: 4, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{1.0, 2.0, 3.0, 4.0}, + }, + }, + }, + { + Dim: 4, + Data: &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: []float32{5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}, + }, + }, + }, + }, + }, + }, + } + + arrowSchema, err := ConvertToArrowSchema(schema) + assert.NoError(t, err) + assert.NotNil(t, arrowSchema) + + recordBuilder := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) + defer recordBuilder.Release() + + err = BuildRecord(recordBuilder, insertData, schema) + assert.NoError(t, err) + + record := recordBuilder.NewRecord() + defer record.Release() + + assert.Equal(t, int64(2), record.NumRows()) + assert.Equal(t, int64(1), record.NumCols()) + + field := arrowSchema.Field(0) + assert.True(t, field.HasMetadata()) + + elementTypeStr, ok := field.Metadata.GetValue("elementType") + assert.True(t, ok) + assert.Equal(t, "101", elementTypeStr) // FloatVector = 101 + + dimStr, ok := field.Metadata.GetValue("dim") + assert.True(t, ok) + assert.Equal(t, "4", dimStr) +} diff --git a/pkg/proto/index_cgo_msg.proto b/pkg/proto/index_cgo_msg.proto index dd08b25143..9f3c442e08 100644 --- a/pkg/proto/index_cgo_msg.proto +++ b/pkg/proto/index_cgo_msg.proto @@ -66,6 +66,7 @@ message OptionalFieldInfo { string field_name = 2; int32 field_type = 3; repeated string data_paths = 4; + int32 element_type = 5; } message BuildIndexInfo { diff --git a/pkg/proto/index_coord.proto b/pkg/proto/index_coord.proto index 46e1183239..a5572f7a84 100644 --- a/pkg/proto/index_coord.proto +++ b/pkg/proto/index_coord.proto @@ -234,6 +234,7 @@ message OptionalFieldInfo { int32 field_type = 3; repeated string data_paths = 4; repeated int64 data_ids = 5; + int32 element_type = 6; } message JobInfo { diff --git a/pkg/proto/indexcgopb/index_cgo_msg.pb.go b/pkg/proto/indexcgopb/index_cgo_msg.pb.go index 29ebd1a422..14550f6cde 100644 --- a/pkg/proto/indexcgopb/index_cgo_msg.pb.go +++ b/pkg/proto/indexcgopb/index_cgo_msg.pb.go @@ -574,10 +574,11 @@ type OptionalFieldInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` - FieldName string `protobuf:"bytes,2,opt,name=field_name,json=fieldName,proto3" json:"field_name,omitempty"` - FieldType int32 `protobuf:"varint,3,opt,name=field_type,json=fieldType,proto3" json:"field_type,omitempty"` - DataPaths []string `protobuf:"bytes,4,rep,name=data_paths,json=dataPaths,proto3" json:"data_paths,omitempty"` + FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` + FieldName string `protobuf:"bytes,2,opt,name=field_name,json=fieldName,proto3" json:"field_name,omitempty"` + FieldType int32 `protobuf:"varint,3,opt,name=field_type,json=fieldType,proto3" json:"field_type,omitempty"` + DataPaths []string `protobuf:"bytes,4,rep,name=data_paths,json=dataPaths,proto3" json:"data_paths,omitempty"` + ElementType int32 `protobuf:"varint,5,opt,name=element_type,json=elementType,proto3" json:"element_type,omitempty"` } func (x *OptionalFieldInfo) Reset() { @@ -640,6 +641,13 @@ func (x *OptionalFieldInfo) GetDataPaths() []string { return nil } +func (x *OptionalFieldInfo) GetElementType() int32 { + if x != nil { + return x.ElementType + } + return 0 +} + type BuildIndexInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1285,7 +1293,7 @@ var file_index_cgo_msg_proto_rawDesc = []byte{ 0x28, 0x09, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x43, 0x41, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x11, 0x47, 0x63, 0x70, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x47, 0x63, 0x70, 0x43, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x4a, 0x53, 0x4f, 0x4e, 0x22, 0x8a, 0x01, 0x0a, 0x11, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x4a, 0x53, 0x4f, 0x4e, 0x22, 0xad, 0x01, 0x0a, 0x11, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, @@ -1294,163 +1302,165 @@ var file_index_cgo_msg_proto_rawDesc = []byte{ 0x65, 0x6c, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, - 0x61, 0x74, 0x61, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xf8, 0x0b, 0x0a, 0x0e, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x61, 0x74, 0x61, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, + 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xf8, 0x0b, 0x0a, 0x0e, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, + 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x09, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x32, 0x0a, 0x15, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x10, + 0x0a, 0x03, 0x64, 0x69, 0x6d, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x64, 0x69, 0x6d, + 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x21, 0x0a, 0x0c, + 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, + 0x43, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, + 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x12, 0x4b, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, + 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x63, 0x67, 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x44, 0x0a, 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4b, 0x65, + 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x52, 0x0b, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, + 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, + 0x6f, 0x6e, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x52, + 0x0a, 0x74, 0x79, 0x70, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x28, 0x0a, 0x10, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, 0x0a, 0x6f, 0x70, 0x74, + 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, + 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x63, 0x67, 0x6f, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x6f, 0x70, 0x74, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x12, 0x36, 0x0a, 0x17, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x14, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x15, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, + 0x79, 0x49, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x1c, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x5f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x19, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x1d, 0x6a, + 0x73, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x74, 0x61, + 0x6e, 0x74, 0x69, 0x76, 0x79, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x54, 0x61, 0x6e, 0x74, 0x69, 0x76, 0x79, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x28, 0x0a, + 0x10, 0x6c, 0x61, 0x63, 0x6b, 0x5f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x5f, 0x72, 0x6f, 0x77, + 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6c, 0x61, 0x63, 0x6b, 0x42, 0x69, 0x6e, + 0x6c, 0x6f, 0x67, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x18, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x5b, 0x0a, 0x14, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x73, 0x65, + 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, + 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x63, 0x67, 0x6f, 0x2e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, + 0x73, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x12, 0x73, 0x65, 0x67, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x61, 0x0a, + 0x16, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x63, 0x67, 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x75, + 0x67, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x14, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x12, 0x46, 0x0a, 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x6d, + 0x61, 0x78, 0x5f, 0x73, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1c, 0x6a, 0x73, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x4d, 0x61, 0x78, 0x53, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, + 0x67, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x4e, 0x0a, 0x24, 0x6a, 0x73, 0x6f, 0x6e, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x73, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, + 0x5f, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, + 0x18, 0x1c, 0x20, 0x01, 0x28, 0x01, 0x52, 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x53, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x61, 0x74, 0x69, 0x6f, 0x54, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3c, 0x0a, 0x1b, 0x6a, 0x73, 0x6f, 0x6e, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x61, 0x74, + 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x6a, + 0x73, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, 0x61, 0x74, + 0x63, 0x68, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x90, 0x01, 0x0a, 0x14, 0x53, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, + 0x2c, 0x0a, 0x12, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x7a, 0x6f, + 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x65, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x23, 0x0a, + 0x0d, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, + 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0xe0, 0x02, 0x0a, 0x11, 0x4c, 0x6f, + 0x61, 0x64, 0x54, 0x65, 0x78, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x18, 0x0a, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x14, 0x0a, + 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x22, 0x0a, + 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x44, 0x12, 0x46, 0x0a, 0x0d, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6f, + 0x72, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, + 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x52, 0x0c, 0x6c, + 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x6d, 0x61, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x6d, 0x61, 0x70, 0x22, 0x87, 0x03, 0x0a, + 0x14, 0x4c, 0x6f, 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, + 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, + 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x67, - 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, - 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, - 0x69, 0x6d, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x64, 0x69, 0x6d, 0x12, 0x2a, 0x0a, - 0x11, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x66, - 0x69, 0x78, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x46, - 0x69, 0x6c, 0x65, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, - 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0b, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0c, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x12, 0x4b, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x63, 0x67, - 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x44, - 0x0a, 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0e, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x52, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, - 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x52, 0x0a, 0x74, 0x79, - 0x70, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, 0x0a, 0x6f, 0x70, 0x74, 0x5f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x69, 0x6c, - 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x63, - 0x67, 0x6f, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x6f, 0x70, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, - 0x36, 0x0a, 0x17, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, - 0x5f, 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x15, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x49, 0x73, - 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x1c, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, 0x52, 0x19, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x63, 0x61, 0x6c, 0x61, 0x72, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x1d, 0x6a, 0x73, 0x6f, 0x6e, - 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x74, 0x61, 0x6e, 0x74, 0x69, - 0x76, 0x79, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73, 0x54, 0x61, 0x6e, - 0x74, 0x69, 0x76, 0x79, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x61, - 0x63, 0x6b, 0x5f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x17, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6c, 0x61, 0x63, 0x6b, 0x42, 0x69, 0x6e, 0x6c, 0x6f, 0x67, - 0x52, 0x6f, 0x77, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x18, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, - 0x14, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x5f, - 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6d, 0x69, - 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x63, 0x67, 0x6f, 0x2e, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x73, 0x65, 0x72, - 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x12, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, - 0x6e, 0x73, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x61, 0x0a, 0x16, 0x73, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x69, 0x6c, - 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x63, - 0x67, 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x14, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x46, 0x0a, - 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x6d, 0x61, 0x78, 0x5f, - 0x73, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x1c, 0x6a, 0x73, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x4d, 0x61, 0x78, 0x53, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x4e, 0x0a, 0x24, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x5f, 0x73, 0x68, 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x1c, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x20, 0x6a, 0x73, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x68, - 0x72, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x61, 0x74, 0x69, 0x6f, 0x54, 0x68, 0x72, 0x65, - 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3c, 0x0a, 0x1b, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, - 0x73, 0x69, 0x7a, 0x65, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x6a, 0x73, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x53, - 0x69, 0x7a, 0x65, 0x22, 0x90, 0x01, 0x0a, 0x14, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x50, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x2c, 0x0a, 0x12, - 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, - 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0xe0, 0x02, 0x0a, 0x11, 0x4c, 0x6f, 0x61, 0x64, 0x54, - 0x65, 0x78, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, - 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x12, 0x38, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, - 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, - 0x12, 0x46, 0x0a, 0x0d, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, - 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, - 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x52, 0x0c, 0x6c, 0x6f, 0x61, 0x64, - 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x5f, 0x6d, 0x6d, 0x61, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x6d, 0x61, 0x70, 0x22, 0x87, 0x03, 0x0a, 0x14, 0x4c, 0x6f, - 0x61, 0x64, 0x4a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, - 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x07, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, - 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, - 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x46, 0x0a, 0x0d, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, - 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, - 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, - 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, - 0x52, 0x0c, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1f, - 0x0a, 0x0b, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x6d, 0x61, 0x70, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x6d, 0x61, 0x70, 0x12, - 0x22, 0x0a, 0x0d, 0x6d, 0x6d, 0x61, 0x70, 0x5f, 0x64, 0x69, 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x6d, 0x61, 0x70, 0x44, 0x69, 0x72, 0x50, - 0x61, 0x74, 0x68, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x63, 0x67, 0x6f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x46, 0x0a, 0x0d, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x52, 0x0c, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, + 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x6d, 0x61, 0x70, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x6d, + 0x61, 0x70, 0x12, 0x22, 0x0a, 0x0d, 0x6d, 0x6d, 0x61, 0x70, 0x5f, 0x64, 0x69, 0x72, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x6d, 0x61, 0x70, 0x44, + 0x69, 0x72, 0x50, 0x61, 0x74, 0x68, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, + 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x63, 0x67, 0x6f, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/proto/indexpb/index_coord.pb.go b/pkg/proto/indexpb/index_coord.pb.go index 21511a0a3a..0970ae572e 100644 --- a/pkg/proto/indexpb/index_coord.pb.go +++ b/pkg/proto/indexpb/index_coord.pb.go @@ -2034,11 +2034,12 @@ type OptionalFieldInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` - FieldName string `protobuf:"bytes,2,opt,name=field_name,json=fieldName,proto3" json:"field_name,omitempty"` - FieldType int32 `protobuf:"varint,3,opt,name=field_type,json=fieldType,proto3" json:"field_type,omitempty"` - DataPaths []string `protobuf:"bytes,4,rep,name=data_paths,json=dataPaths,proto3" json:"data_paths,omitempty"` - DataIds []int64 `protobuf:"varint,5,rep,packed,name=data_ids,json=dataIds,proto3" json:"data_ids,omitempty"` + FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` + FieldName string `protobuf:"bytes,2,opt,name=field_name,json=fieldName,proto3" json:"field_name,omitempty"` + FieldType int32 `protobuf:"varint,3,opt,name=field_type,json=fieldType,proto3" json:"field_type,omitempty"` + DataPaths []string `protobuf:"bytes,4,rep,name=data_paths,json=dataPaths,proto3" json:"data_paths,omitempty"` + DataIds []int64 `protobuf:"varint,5,rep,packed,name=data_ids,json=dataIds,proto3" json:"data_ids,omitempty"` + ElementType int32 `protobuf:"varint,6,opt,name=element_type,json=elementType,proto3" json:"element_type,omitempty"` } func (x *OptionalFieldInfo) Reset() { @@ -2108,6 +2109,13 @@ func (x *OptionalFieldInfo) GetDataIds() []int64 { return nil } +func (x *OptionalFieldInfo) GetElementType() int32 { + if x != nil { + return x.ElementType + } + return 0 +} + type JobInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3160,7 +3168,7 @@ var file_index_coord_proto_rawDesc = []byte{ 0x74, 0x12, 0x2c, 0x0a, 0x11, 0x47, 0x63, 0x70, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x4a, 0x53, 0x4f, 0x4e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x47, 0x63, 0x70, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x4a, 0x53, 0x4f, 0x4e, 0x22, - 0xa5, 0x01, 0x0a, 0x11, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x65, 0x6c, + 0xc8, 0x01, 0x0a, 0x11, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, @@ -3170,217 +3178,219 @@ var file_index_coord_proto_rawDesc = []byte{ 0x0a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, - 0x64, 0x61, 0x74, 0x61, 0x49, 0x64, 0x73, 0x22, 0xcc, 0x01, 0x0a, 0x07, 0x4a, 0x6f, 0x62, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x10, - 0x0a, 0x03, 0x64, 0x69, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x64, 0x69, 0x6d, - 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x0c, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, - 0x61, 0x69, 0x72, 0x52, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x70, 0x6f, 0x64, 0x49, 0x44, 0x22, 0x5e, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x64, 0x61, 0x74, 0x61, 0x49, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xcc, 0x01, 0x0a, 0x07, 0x4a, + 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x6f, + 0x77, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x6f, 0x77, + 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, + 0x64, 0x69, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x44, 0x0a, + 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x52, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x44, 0x22, 0x5e, 0x0a, 0x19, 0x47, 0x65, 0x74, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x69, 0x6e, 0x64, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x91, 0x01, 0x0a, 0x1a, 0x47, 0x65, + 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3e, 0x0a, + 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x22, 0x38, 0x0a, + 0x12, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x91, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3e, 0x0a, 0x0b, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x22, 0x38, 0x0a, 0x12, 0x4c, 0x69, - 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x22, 0x8a, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, - 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x3e, 0x0a, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x22, 0xc2, 0x03, 0x0a, 0x0b, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, - 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x3c, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x22, 0x8a, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x33, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, + 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x3e, 0x0a, 0x0b, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, + 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x49, + 0x6e, 0x66, 0x6f, 0x73, 0x22, 0xc2, 0x03, 0x0a, 0x0b, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, + 0x54, 0x61, 0x73, 0x6b, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x44, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, + 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, + 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x12, 0x32, 0x0a, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, 0x69, 0x6c, + 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, + 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x6d, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x64, + 0x69, 0x6d, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x65, 0x6e, 0x74, 0x72, 0x6f, 0x69, 0x64, 0x73, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x65, 0x6e, 0x74, + 0x72, 0x6f, 0x69, 0x64, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x51, 0x0a, 0x0c, 0x53, 0x65, 0x67, + 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x49, 0x44, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, + 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, + 0x52, 0x6f, 0x77, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x49, 0x44, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x03, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x49, 0x44, 0x73, 0x22, 0x47, 0x0a, 0x0c, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, + 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0xc1, 0x03, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x73, 0x54, + 0x61, 0x73, 0x6b, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x67, + 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, + 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x16, + 0x0a, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x73, 0x18, 0x08, - 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x73, - 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, + 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, - 0x66, 0x61, 0x69, 0x6c, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x66, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x64, 0x69, 0x6d, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x64, 0x69, 0x6d, 0x12, - 0x25, 0x0a, 0x0e, 0x63, 0x65, 0x6e, 0x74, 0x72, 0x6f, 0x69, 0x64, 0x73, 0x5f, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x65, 0x6e, 0x74, 0x72, 0x6f, 0x69, - 0x64, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x51, 0x0a, 0x0c, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x02, 0x49, 0x44, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x6f, - 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x6f, 0x77, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x49, 0x44, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x03, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x49, 0x44, 0x73, 0x22, 0x47, 0x0a, 0x0c, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x49, 0x44, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, - 0x68, 0x73, 0x22, 0xc1, 0x03, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x73, 0x54, 0x61, 0x73, 0x6b, - 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, - 0x74, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x65, 0x67, 0x6d, 0x65, - 0x6e, 0x74, 0x49, 0x44, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x5f, 0x63, - 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, - 0x73, 0x65, 0x72, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x74, - 0x61, 0x73, 0x6b, 0x49, 0x44, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x61, 0x73, - 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, - 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, - 0x6f, 0x64, 0x65, 0x49, 0x44, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x61, 0x69, - 0x6c, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x66, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5f, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, - 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x3f, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x73, 0x53, 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x4a, - 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x6e, 0x52, 0x65, 0x63, - 0x79, 0x63, 0x6c, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x52, - 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x2a, 0x5b, 0x0a, 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x6f, 0x6e, 0x65, - 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x4a, 0x6f, 0x62, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x4a, 0x6f, 0x62, 0x54, 0x79, - 0x70, 0x65, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x4a, 0x6f, 0x62, 0x10, 0x02, 0x12, 0x13, - 0x0a, 0x0f, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x4a, 0x6f, - 0x62, 0x10, 0x03, 0x2a, 0x83, 0x01, 0x0a, 0x08, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x10, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x6e, 0x65, - 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x49, 0x6e, - 0x69, 0x74, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x49, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, - 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, - 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x61, - 0x69, 0x6c, 0x65, 0x64, 0x10, 0x04, 0x12, 0x11, 0x0a, 0x0d, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x74, 0x72, 0x79, 0x10, 0x05, 0x2a, 0x55, 0x0a, 0x0b, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x53, 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, - 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x6f, 0x72, 0x74, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, - 0x54, 0x65, 0x78, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4a, 0x6f, 0x62, 0x10, 0x02, 0x12, 0x0b, - 0x0a, 0x07, 0x42, 0x4d, 0x32, 0x35, 0x4a, 0x6f, 0x62, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4a, - 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4a, 0x6f, 0x62, 0x10, 0x04, - 0x32, 0xf4, 0x09, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x12, - 0x54, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, - 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0a, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x12, 0x25, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x49, 0x6e, + 0x66, 0x61, 0x69, 0x6c, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x66, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x29, 0x0a, + 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, + 0x44, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, + 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x3f, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x4a, + 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, + 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x52, 0x0a, 0x73, + 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x6e, + 0x52, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, + 0x61, 0x6e, 0x52, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x2a, 0x5b, 0x0a, 0x07, 0x4a, 0x6f, 0x62, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x4e, + 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4a, 0x6f, 0x62, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x4a, 0x6f, + 0x62, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x4a, 0x6f, 0x62, 0x10, + 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x4a, 0x6f, 0x62, 0x10, 0x03, 0x2a, 0x83, 0x01, 0x0a, 0x08, 0x4a, 0x6f, 0x62, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x4e, + 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x49, 0x6e, 0x69, 0x74, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x6f, 0x62, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x49, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x02, 0x12, + 0x14, 0x0a, 0x10, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x46, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x65, 0x64, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x04, 0x12, 0x11, 0x0a, 0x0d, 0x4a, 0x6f, 0x62, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x74, 0x72, 0x79, 0x10, 0x05, 0x2a, 0x55, 0x0a, 0x0b, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x75, 0x62, 0x4a, 0x6f, 0x62, 0x12, 0x08, 0x0a, 0x04, 0x4e, + 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x6f, 0x72, 0x74, 0x10, 0x01, 0x12, + 0x10, 0x0a, 0x0c, 0x54, 0x65, 0x78, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4a, 0x6f, 0x62, 0x10, + 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4d, 0x32, 0x35, 0x4a, 0x6f, 0x62, 0x10, 0x03, 0x12, 0x13, + 0x0a, 0x0f, 0x4a, 0x73, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4a, 0x6f, + 0x62, 0x10, 0x04, 0x32, 0xf4, 0x09, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x43, 0x6f, 0x6f, + 0x72, 0x64, 0x12, 0x54, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x12, 0x26, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, + 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0a, 0x41, 0x6c, 0x74, 0x65, + 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x25, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x41, 0x6c, 0x74, 0x65, + 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x66, 0x0a, 0x0d, + 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x28, 0x2e, + 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2f, 0x2e, 0x6d, + 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, + 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x64, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, + 0x6f, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x69, + 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x09, 0x44, 0x72, 0x6f, 0x70, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x12, 0x24, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x66, 0x0a, 0x0d, 0x47, 0x65, 0x74, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x28, 0x2e, 0x6d, 0x69, 0x6c, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x66, 0x0a, 0x0d, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x62, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x28, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, - 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x7b, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, - 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x6d, 0x69, 0x6c, - 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, - 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, - 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x09, 0x44, 0x72, 0x6f, 0x70, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x12, 0x24, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x66, 0x0a, 0x0d, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x62, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x28, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x62, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x49, - 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x75, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, - 0x74, 0x69, 0x63, 0x73, 0x12, 0x2d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x62, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x75, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, + 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x2d, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7e, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7e, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x30, + 0x78, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x12, 0x53, 0x68, 0x6f, 0x77, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, - 0x69, 0x6c, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x12, 0x53, 0x68, 0x6f, 0x77, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x69, - 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, - 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x5f, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x12, 0x26, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x47, - 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x0b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x12, 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x69, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x77, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5f, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x26, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, - 0x73, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, - 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x0b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x27, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, + 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x69, + 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, + 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/tests/integration/datanode/struct_array_test.go b/tests/integration/datanode/struct_array_test.go index 0df492656c..6668e7cdcf 100644 --- a/tests/integration/datanode/struct_array_test.go +++ b/tests/integration/datanode/struct_array_test.go @@ -166,6 +166,7 @@ func (s *ArrayStructDataNodeSuite) loadCollection(collectionName string) { s.NoError(err) err = merr.Error(createIndexStatus) s.NoError(err) + log.Info("=========================Index created for float vector=========================") s.WaitForIndexBuilt(context.TODO(), collectionName, integration.FloatVecField) createIndexResult, err := c.MilvusClient.CreateIndex(context.TODO(), &milvuspb.CreateIndexRequest{ @@ -179,7 +180,7 @@ func (s *ArrayStructDataNodeSuite) loadCollection(collectionName string) { s.Require().Equal(createIndexResult.GetErrorCode(), commonpb.ErrorCode_Success) s.WaitForIndexBuilt(context.TODO(), collectionName, integration.StructSubFloatVecField) - log.Info("=========================Index created=========================") + log.Info("=========================Index created for array of vector=========================") // load loadStatus, err := c.MilvusClient.LoadCollection(context.TODO(), &milvuspb.LoadCollectionRequest{ @@ -314,9 +315,9 @@ func (s *ArrayStructDataNodeSuite) query(collectionName string) { topk := 10 roundDecimal := -1 - params := integration.GetSearchParams(integration.IndexFaissIvfFlat, metric.IP) + params := integration.GetSearchParams(integration.IndexEmbListHNSW, metric.MaxSim) searchReq := integration.ConstructEmbeddingListSearchRequest("", collectionName, expr, - integration.StructSubFloatVecField, schemapb.DataType_FloatVector, nil, metric.MaxSim, params, nq, s.dim, topk, roundDecimal) + integration.StructSubFloatVecField, schemapb.DataType_FloatVector, []string{integration.StructArrayField}, metric.MaxSim, params, nq, s.dim, topk, roundDecimal) searchResult, _ := c.MilvusClient.Search(context.TODO(), searchReq)