// Copyright (C) 2019-2020 Zilliz. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License #include #include #include "common/JsonUtils.h" #include "index/JsonIndexBuilder.h" #include "simdjson/error.h" namespace milvus::index { template void ProcessJsonFieldData( const std::vector>& field_datas, const proto::schema::FieldSchema& schema, const std::string& nested_path, const JsonCastType& cast_type, JsonCastFunction cast_function, JsonDataAdder data_adder, JsonNullAdder null_adder, JsonNonExistAdder non_exist_adder, JsonErrorRecorder error_recorder) { int64_t offset = 0; using SIMDJSON_T = std::conditional_t, std::string_view, T>; auto tokens = parse_json_pointer(nested_path); bool is_array = cast_type.data_type() == JsonCastType::DataType::ARRAY; for (const auto& data : field_datas) { auto n = data->get_num_rows(); for (int64_t i = 0; i < n; i++) { auto json_column = static_cast(data->RawValue(i)); if (schema.nullable() && !data->is_valid(i)) { non_exist_adder(offset); null_adder(offset); data_adder(nullptr, 0, offset++); continue; } auto exists = path_exists(json_column->dom_doc(), tokens); if (!exists || !json_column->exist(nested_path)) { error_recorder( *json_column, nested_path, simdjson::NO_SUCH_FIELD); non_exist_adder(offset); data_adder(nullptr, 0, offset++); continue; } folly::fbvector values; if (is_array) { auto doc = json_column->dom_doc(); auto array_res = doc.at_pointer(nested_path).get_array(); if (array_res.error() != simdjson::SUCCESS) { error_recorder( *json_column, nested_path, array_res.error()); } else { auto array_values = array_res.value(); for (auto value : array_values) { auto val = value.template get(); if (val.error() == simdjson::SUCCESS) { values.push_back(static_cast(val.value())); } } } } else { if (cast_function.match()) { auto res = JsonCastFunction::CastJsonValue( cast_function, *json_column, nested_path); if (res.has_value()) { values.push_back(res.value()); } } else { value_result res = json_column->at(nested_path); if (res.error() != simdjson::SUCCESS) { error_recorder(*json_column, nested_path, res.error()); } else { values.push_back(static_cast(res.value())); } } } data_adder(values.data(), values.size(), offset++); } } } template void ProcessJsonFieldData( const std::vector>& field_datas, const proto::schema::FieldSchema& schema, const std::string& nested_path, const JsonCastType& cast_type, JsonCastFunction cast_function, JsonDataAdder data_adder, JsonNullAdder null_adder, JsonNonExistAdder non_exist_adder, JsonErrorRecorder error_recorder); template void ProcessJsonFieldData( const std::vector>& field_datas, const proto::schema::FieldSchema& schema, const std::string& nested_path, const JsonCastType& cast_type, JsonCastFunction cast_function, JsonDataAdder data_adder, JsonNullAdder null_adder, JsonNonExistAdder non_exist_adder, JsonErrorRecorder error_recorder); template void ProcessJsonFieldData( const std::vector>& field_datas, const proto::schema::FieldSchema& schema, const std::string& nested_path, const JsonCastType& cast_type, JsonCastFunction cast_function, JsonDataAdder data_adder, JsonNullAdder null_adder, JsonNonExistAdder non_exist_adder, JsonErrorRecorder error_recorder); template void ProcessJsonFieldData( const std::vector>& field_datas, const proto::schema::FieldSchema& schema, const std::string& nested_path, const JsonCastType& cast_type, JsonCastFunction cast_function, JsonDataAdder data_adder, JsonNullAdder null_adder, JsonNonExistAdder non_exist_adder, JsonErrorRecorder error_recorder); } // namespace milvus::index