// Copyright (C) 2019-2020 Zilliz. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License #include "SkipIndex.h" #include "cachinglayer/CacheSlot.h" #include "cachinglayer/Utils.h" namespace milvus { static const FieldChunkMetrics defaultFieldChunkMetrics; const cachinglayer::PinWrapper SkipIndex::GetFieldChunkMetrics(milvus::FieldId field_id, int chunk_id) const { // skip index structure must be setup before using, thus we do not lock here. auto field_metrics = fieldChunkMetrics_.find(field_id); if (field_metrics != fieldChunkMetrics_.end()) { auto& field_chunk_metrics = field_metrics->second; auto ca = cachinglayer::SemiInlineGet( field_chunk_metrics->PinCells(nullptr, {chunk_id})); auto metrics = ca->get_cell_of(chunk_id); return cachinglayer::PinWrapper(ca, metrics); } return cachinglayer::PinWrapper( &defaultFieldChunkMetrics); } std::vector< std::pair>> FieldChunkMetricsTranslator::get_cells( const std::vector& cids) { std::vector>> cells; cells.reserve(cids.size()); if (data_type_ == milvus::DataType::VARCHAR) { for (auto chunk_id : cids) { auto pw = column_->GetChunk(nullptr, chunk_id); auto chunk = static_cast(pw.get()); int num_rows = chunk->RowNums(); auto chunk_metrics = std::make_unique(); if (num_rows > 0) { auto info = ProcessStringFieldMetrics(chunk); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; } chunk_metrics->hasValue_ = chunk_metrics->null_count_ != num_rows; cells.emplace_back(chunk_id, std::move(chunk_metrics)); } } else { for (auto chunk_id : cids) { auto pw = column_->GetChunk(nullptr, chunk_id); auto chunk = static_cast(pw.get()); auto span = chunk->Span(); const void* chunk_data = span.data(); const bool* valid_data = span.valid_data(); int64_t count = span.row_count(); auto chunk_metrics = std::make_unique(); if (count > 0) { switch (data_type_) { case DataType::INT8: { const int8_t* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } case DataType::INT16: { const int16_t* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } case DataType::INT32: { const int32_t* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } case DataType::INT64: { const int64_t* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } case DataType::FLOAT: { const float* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } case DataType::DOUBLE: { const double* typedData = static_cast(chunk_data); auto info = ProcessFieldMetrics( typedData, valid_data, count); chunk_metrics->min_ = Metrics(info.min_); chunk_metrics->max_ = Metrics(info.max_); chunk_metrics->null_count_ = info.null_count_; break; } } } chunk_metrics->hasValue_ = chunk_metrics->null_count_ != count; cells.emplace_back(chunk_id, std::move(chunk_metrics)); } } return cells; } } // namespace milvus