enhance: optimize term expr performance (#45490)

issue: https://github.com/milvus-io/milvus/issues/45641
pr: https://github.com/milvus-io/milvus/pull/45491

---------

Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
Bingyi Sun 2025-11-19 11:51:06 +08:00 committed by GitHub
parent e695d8a1d0
commit f1844c9841
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 36 additions and 29 deletions

View File

@ -139,7 +139,6 @@ InitDefaultDeleteDumpBatchSize(int32_t val) {
val);
}
void
InitTrace(CTraceConfig* config) {
auto traceConfig = milvus::tracer::TraceConfig{config->exporter,

View File

@ -198,7 +198,7 @@ class FlatVectorElement : public MultiElement {
In(const ValueType& value) const override {
if (std::holds_alternative<T>(value)) {
for (const auto& v : values_) {
if (v == value)
if (v == std::get<T>(value))
return true;
}
}

View File

@ -908,7 +908,7 @@ class SegmentExpr : public Expr {
template <typename T, typename FUNC, typename... ValTypes>
VectorPtr
ProcessIndexChunks(FUNC func, ValTypes... values) {
ProcessIndexChunks(FUNC func, const ValTypes&... values) {
typedef std::
conditional_t<std::is_same_v<T, std::string_view>, std::string, T>
IndexInnerType;

View File

@ -66,7 +66,6 @@ class PhyGISFunctionFilterExpr : public SegmentExpr {
}
}
private:
VectorPtr
EvalForIndexSegment();

View File

@ -810,6 +810,7 @@ PhyTermFilterExpr::ExecVisitorImplForIndex() {
return nullptr;
}
if (!arg_inited_) {
std::vector<IndexInnerType> vals;
for (auto& val : expr_->vals_) {
if constexpr (std::is_same_v<T, double>) {
@ -822,17 +823,23 @@ PhyTermFilterExpr::ExecVisitorImplForIndex() {
// Generic overflow handling for all types
bool overflowed = false;
auto converted_val = GetValueFromProtoWithOverflow<T>(val, overflowed);
auto converted_val =
GetValueFromProtoWithOverflow<T>(val, overflowed);
if (!overflowed) {
vals.emplace_back(converted_val);
}
}
arg_set_ = std::make_shared<FlatVectorElement<IndexInnerType>>(vals);
arg_inited_ = true;
}
auto execute_sub_batch = [](Index* index_ptr,
const std::vector<IndexInnerType>& vals) {
TermIndexFunc<T> func;
return func(index_ptr, vals.size(), vals.data());
};
auto res = ProcessIndexChunks<T>(execute_sub_batch, vals);
auto args =
std::dynamic_pointer_cast<FlatVectorElement<IndexInnerType>>(arg_set_);
auto res = ProcessIndexChunks<T>(execute_sub_batch, args->values_);
AssertInfo(res->size() == real_batch_size,
"internal error: expr processed rows {} not equal "
"expect batch size {}",

View File

@ -214,7 +214,8 @@ class DeletedRecord {
SortedDeleteList::Accessor accessor(deleted_lists_);
int total_size = accessor.size();
while (total_size - dumped_entry_count_.load() > DELETE_DUMP_BATCH_SIZE) {
while (total_size - dumped_entry_count_.load() >
DELETE_DUMP_BATCH_SIZE) {
int32_t bitsize = 0;
if constexpr (is_sealed) {
bitsize = sealed_row_count_;
@ -232,11 +233,14 @@ class DeletedRecord {
snapshots_.back().second.size());
}
while (total_size - dumped_entry_count_.load() > DELETE_DUMP_BATCH_SIZE &&
while (total_size - dumped_entry_count_.load() >
DELETE_DUMP_BATCH_SIZE &&
it != accessor.end()) {
Timestamp dump_ts = 0;
for (auto size = 0; size < DELETE_DUMP_BATCH_SIZE && it != accessor.end(); ++it, ++size) {
for (auto size = 0;
size < DELETE_DUMP_BATCH_SIZE && it != accessor.end();
++it, ++size) {
bitmap.set(it->second);
dump_ts = it->first;
}

View File

@ -294,8 +294,7 @@ class FileWriter {
// for global configuration
static WriteMode
mode_; // The write mode, which can be 'buffered' (default) or 'direct'.
static size_t
buffer_size_;
static size_t buffer_size_;
// for rate limiter
io::Priority priority_;

View File

@ -343,8 +343,7 @@ GenerateRandomSparseFloatVector(size_t rows,
return tensor;
}
inline std::string
generateRandomPoint() {
inline std::string generateRandomPoint() {
return "POINT(" +
std::to_string(static_cast<double>(rand()) / RAND_MAX * 360.0 -
180.0) +