mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-28 22:45:26 +08:00
389104d200
issue: #41435 this is to prevent AI from thinking of our exception throwing as a dangerous PANIC operation that terminates the program. Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
116 lines
4.2 KiB
C++
116 lines
4.2 KiB
C++
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "storage/InsertData.h"
|
|
#include "storage/Event.h"
|
|
#include "storage/Util.h"
|
|
#include "common/Json.h"
|
|
#include "common/FieldMeta.h"
|
|
#include "common/Consts.h"
|
|
|
|
namespace milvus::storage {
|
|
|
|
void
|
|
InsertData::SetFieldDataMeta(const FieldDataMeta& meta) {
|
|
AssertInfo(!field_data_meta_.has_value(), "field meta has been inited");
|
|
field_data_meta_ = meta;
|
|
}
|
|
|
|
std::vector<uint8_t>
|
|
InsertData::Serialize(StorageType medium) {
|
|
switch (medium) {
|
|
case StorageType::Remote:
|
|
return serialize_to_remote_file();
|
|
case StorageType::LocalDisk:
|
|
return serialize_to_local_file();
|
|
default:
|
|
ThrowInfo(DataFormatBroken,
|
|
fmt::format("unsupported medium type {}", medium));
|
|
}
|
|
}
|
|
|
|
// TODO :: handle string and bool type
|
|
std::vector<uint8_t>
|
|
InsertData::serialize_to_remote_file() {
|
|
AssertInfo(field_data_meta_.has_value(), "field data not exist");
|
|
AssertInfo(payload_reader_->has_field_data(), "empty field data");
|
|
auto field_data = payload_reader_->get_field_data();
|
|
DataType data_type = field_data->get_data_type();
|
|
|
|
// create descriptor event
|
|
DescriptorEvent descriptor_event;
|
|
auto& des_event_data = descriptor_event.event_data;
|
|
auto& des_fix_part = des_event_data.fix_part;
|
|
des_fix_part.collection_id = field_data_meta_->collection_id;
|
|
des_fix_part.partition_id = field_data_meta_->partition_id;
|
|
des_fix_part.segment_id = field_data_meta_->segment_id;
|
|
des_fix_part.field_id = field_data_meta_->field_id;
|
|
des_fix_part.start_timestamp = time_range_.first;
|
|
des_fix_part.end_timestamp = time_range_.second;
|
|
des_fix_part.data_type = milvus::proto::schema::DataType(data_type);
|
|
for (auto i = int8_t(EventType::DescriptorEvent);
|
|
i < int8_t(EventType::EventTypeEnd);
|
|
i++) {
|
|
des_event_data.post_header_lengths.push_back(
|
|
GetEventFixPartSize(EventType(i)));
|
|
}
|
|
des_event_data.extras[ORIGIN_SIZE_KEY] = std::to_string(field_data->Size());
|
|
des_event_data.extras[NULLABLE] = field_data->IsNullable();
|
|
|
|
auto& des_event_header = descriptor_event.event_header;
|
|
// TODO :: set timestamp
|
|
des_event_header.timestamp_ = 0;
|
|
|
|
// serialize descriptor event data
|
|
auto des_event_bytes = descriptor_event.Serialize();
|
|
|
|
// create insert event
|
|
InsertEvent insert_event;
|
|
insert_event.event_offset = des_event_bytes.size();
|
|
auto& insert_event_data = insert_event.event_data;
|
|
insert_event_data.start_timestamp = time_range_.first;
|
|
insert_event_data.end_timestamp = time_range_.second;
|
|
insert_event_data.payload_reader = payload_reader_;
|
|
|
|
auto& insert_event_header = insert_event.event_header;
|
|
// TODO :: set timestamps
|
|
insert_event_header.timestamp_ = 0;
|
|
insert_event_header.event_type_ = EventType::InsertEvent;
|
|
|
|
// serialize insert event
|
|
auto insert_event_bytes = insert_event.Serialize();
|
|
|
|
des_event_bytes.insert(des_event_bytes.end(),
|
|
insert_event_bytes.begin(),
|
|
insert_event_bytes.end());
|
|
|
|
return des_event_bytes;
|
|
}
|
|
|
|
// local insert file format
|
|
// -------------------------------------------
|
|
// | Rows(int) | Dimension(int) | InsertData |
|
|
// -------------------------------------------
|
|
std::vector<uint8_t>
|
|
InsertData::serialize_to_local_file() {
|
|
LocalInsertEvent event;
|
|
event.field_data = GetFieldData();
|
|
|
|
return event.Serialize();
|
|
}
|
|
|
|
} // namespace milvus::storage
|