mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 01:28:27 +08:00
enhance: optimize term expr performance (#45490)
issue: https://github.com/milvus-io/milvus/issues/45641 pr: https://github.com/milvus-io/milvus/pull/45491 --------- Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
parent
e695d8a1d0
commit
f1844c9841
@ -139,7 +139,6 @@ InitDefaultDeleteDumpBatchSize(int32_t val) {
|
|||||||
val);
|
val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
InitTrace(CTraceConfig* config) {
|
InitTrace(CTraceConfig* config) {
|
||||||
auto traceConfig = milvus::tracer::TraceConfig{config->exporter,
|
auto traceConfig = milvus::tracer::TraceConfig{config->exporter,
|
||||||
|
|||||||
@ -198,7 +198,7 @@ class FlatVectorElement : public MultiElement {
|
|||||||
In(const ValueType& value) const override {
|
In(const ValueType& value) const override {
|
||||||
if (std::holds_alternative<T>(value)) {
|
if (std::holds_alternative<T>(value)) {
|
||||||
for (const auto& v : values_) {
|
for (const auto& v : values_) {
|
||||||
if (v == value)
|
if (v == std::get<T>(value))
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -908,7 +908,7 @@ class SegmentExpr : public Expr {
|
|||||||
|
|
||||||
template <typename T, typename FUNC, typename... ValTypes>
|
template <typename T, typename FUNC, typename... ValTypes>
|
||||||
VectorPtr
|
VectorPtr
|
||||||
ProcessIndexChunks(FUNC func, ValTypes... values) {
|
ProcessIndexChunks(FUNC func, const ValTypes&... values) {
|
||||||
typedef std::
|
typedef std::
|
||||||
conditional_t<std::is_same_v<T, std::string_view>, std::string, T>
|
conditional_t<std::is_same_v<T, std::string_view>, std::string, T>
|
||||||
IndexInnerType;
|
IndexInnerType;
|
||||||
|
|||||||
@ -66,7 +66,6 @@ class PhyGISFunctionFilterExpr : public SegmentExpr {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
VectorPtr
|
VectorPtr
|
||||||
EvalForIndexSegment();
|
EvalForIndexSegment();
|
||||||
|
|||||||
@ -810,6 +810,7 @@ PhyTermFilterExpr::ExecVisitorImplForIndex() {
|
|||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!arg_inited_) {
|
||||||
std::vector<IndexInnerType> vals;
|
std::vector<IndexInnerType> vals;
|
||||||
for (auto& val : expr_->vals_) {
|
for (auto& val : expr_->vals_) {
|
||||||
if constexpr (std::is_same_v<T, double>) {
|
if constexpr (std::is_same_v<T, double>) {
|
||||||
@ -822,17 +823,23 @@ PhyTermFilterExpr::ExecVisitorImplForIndex() {
|
|||||||
|
|
||||||
// Generic overflow handling for all types
|
// Generic overflow handling for all types
|
||||||
bool overflowed = false;
|
bool overflowed = false;
|
||||||
auto converted_val = GetValueFromProtoWithOverflow<T>(val, overflowed);
|
auto converted_val =
|
||||||
|
GetValueFromProtoWithOverflow<T>(val, overflowed);
|
||||||
if (!overflowed) {
|
if (!overflowed) {
|
||||||
vals.emplace_back(converted_val);
|
vals.emplace_back(converted_val);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
arg_set_ = std::make_shared<FlatVectorElement<IndexInnerType>>(vals);
|
||||||
|
arg_inited_ = true;
|
||||||
|
}
|
||||||
auto execute_sub_batch = [](Index* index_ptr,
|
auto execute_sub_batch = [](Index* index_ptr,
|
||||||
const std::vector<IndexInnerType>& vals) {
|
const std::vector<IndexInnerType>& vals) {
|
||||||
TermIndexFunc<T> func;
|
TermIndexFunc<T> func;
|
||||||
return func(index_ptr, vals.size(), vals.data());
|
return func(index_ptr, vals.size(), vals.data());
|
||||||
};
|
};
|
||||||
auto res = ProcessIndexChunks<T>(execute_sub_batch, vals);
|
auto args =
|
||||||
|
std::dynamic_pointer_cast<FlatVectorElement<IndexInnerType>>(arg_set_);
|
||||||
|
auto res = ProcessIndexChunks<T>(execute_sub_batch, args->values_);
|
||||||
AssertInfo(res->size() == real_batch_size,
|
AssertInfo(res->size() == real_batch_size,
|
||||||
"internal error: expr processed rows {} not equal "
|
"internal error: expr processed rows {} not equal "
|
||||||
"expect batch size {}",
|
"expect batch size {}",
|
||||||
|
|||||||
@ -214,7 +214,8 @@ class DeletedRecord {
|
|||||||
SortedDeleteList::Accessor accessor(deleted_lists_);
|
SortedDeleteList::Accessor accessor(deleted_lists_);
|
||||||
int total_size = accessor.size();
|
int total_size = accessor.size();
|
||||||
|
|
||||||
while (total_size - dumped_entry_count_.load() > DELETE_DUMP_BATCH_SIZE) {
|
while (total_size - dumped_entry_count_.load() >
|
||||||
|
DELETE_DUMP_BATCH_SIZE) {
|
||||||
int32_t bitsize = 0;
|
int32_t bitsize = 0;
|
||||||
if constexpr (is_sealed) {
|
if constexpr (is_sealed) {
|
||||||
bitsize = sealed_row_count_;
|
bitsize = sealed_row_count_;
|
||||||
@ -232,11 +233,14 @@ class DeletedRecord {
|
|||||||
snapshots_.back().second.size());
|
snapshots_.back().second.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
while (total_size - dumped_entry_count_.load() > DELETE_DUMP_BATCH_SIZE &&
|
while (total_size - dumped_entry_count_.load() >
|
||||||
|
DELETE_DUMP_BATCH_SIZE &&
|
||||||
it != accessor.end()) {
|
it != accessor.end()) {
|
||||||
Timestamp dump_ts = 0;
|
Timestamp dump_ts = 0;
|
||||||
|
|
||||||
for (auto size = 0; size < DELETE_DUMP_BATCH_SIZE && it != accessor.end(); ++it, ++size) {
|
for (auto size = 0;
|
||||||
|
size < DELETE_DUMP_BATCH_SIZE && it != accessor.end();
|
||||||
|
++it, ++size) {
|
||||||
bitmap.set(it->second);
|
bitmap.set(it->second);
|
||||||
dump_ts = it->first;
|
dump_ts = it->first;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -294,8 +294,7 @@ class FileWriter {
|
|||||||
// for global configuration
|
// for global configuration
|
||||||
static WriteMode
|
static WriteMode
|
||||||
mode_; // The write mode, which can be 'buffered' (default) or 'direct'.
|
mode_; // The write mode, which can be 'buffered' (default) or 'direct'.
|
||||||
static size_t
|
static size_t buffer_size_;
|
||||||
buffer_size_;
|
|
||||||
|
|
||||||
// for rate limiter
|
// for rate limiter
|
||||||
io::Priority priority_;
|
io::Priority priority_;
|
||||||
|
|||||||
@ -343,8 +343,7 @@ GenerateRandomSparseFloatVector(size_t rows,
|
|||||||
return tensor;
|
return tensor;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline std::string
|
inline std::string generateRandomPoint() {
|
||||||
generateRandomPoint() {
|
|
||||||
return "POINT(" +
|
return "POINT(" +
|
||||||
std::to_string(static_cast<double>(rand()) / RAND_MAX * 360.0 -
|
std::to_string(static_cast<double>(rand()) / RAND_MAX * 360.0 -
|
||||||
180.0) +
|
180.0) +
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user