enhance: improve observability in trace for segcore scalar expression (#44260)

Ref https://github.com/milvus-io/milvus/issues/44259

This PR connects the trace between go and segcore, and add full traces
for scalar expression calling chain:
<img width="2418" height="960" alt="image"
src="https://github.com/user-attachments/assets/8cad69d7-bcb7-4002-a4e3-679a3641e229"
/>
<img width="2452" height="850" alt="image"
src="https://github.com/user-attachments/assets/8b44aed0-0f03-48a7-baa0-b022fee994ce"
/>
<img width="2403" height="707" alt="image"
src="https://github.com/user-attachments/assets/cd6f0601-0d5c-4087-8ed8-2385f1bc740b"
/>

---------

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>

internal/core/src/common/Types.h

@@ -884,6 +884,9 @@ struct fmt::formatter<milvus::OpType> : formatter<string_view> {
             case milvus::OpType::PhraseMatch:
                 name = "PhraseMatch";
                 break;
+            case milvus::OpType::InnerMatch:
+                name = "InnerMatch";
+                break;
         }
         return formatter<string_view>::format(name, ctx);
     }

internal/core/src/exec/Driver.cpp

@@ -15,6 +15,8 @@
 // limitations under the License.
 
 #include "Driver.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include <cassert>
 #include <memory>
@@ -57,41 +59,49 @@ DriverFactory::CreateDriver(std::unique_ptr<DriverContext> ctx,
         if (auto filterbitsnode =
                 std::dynamic_pointer_cast<const plan::FilterBitsNode>(
                     plannode)) {
+            tracer::AddEvent("create_operator: FilterBitsNode");
             operators.push_back(std::make_unique<PhyFilterBitsNode>(
                 id, ctx.get(), filterbitsnode));
         } else if (auto filternode =
                        std::dynamic_pointer_cast<const plan::FilterNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: FilterNode");
             operators.push_back(std::make_unique<PhyIterativeFilterNode>(
                 id, ctx.get(), filternode));
         } else if (auto mvccnode =
                        std::dynamic_pointer_cast<const plan::MvccNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: MvccNode");
             operators.push_back(
                 std::make_unique<PhyMvccNode>(id, ctx.get(), mvccnode));
         } else if (auto countnode =
                        std::dynamic_pointer_cast<const plan::CountNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: CountNode");
             operators.push_back(
                 std::make_unique<PhyCountNode>(id, ctx.get(), countnode));
         } else if (auto vectorsearchnode =
                        std::dynamic_pointer_cast<const plan::VectorSearchNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: VectorSearchNode");
             operators.push_back(std::make_unique<PhyVectorSearchNode>(
                 id, ctx.get(), vectorsearchnode));
         } else if (auto groupbynode =
                        std::dynamic_pointer_cast<const plan::GroupByNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: GroupByNode");
             operators.push_back(
                 std::make_unique<PhyGroupByNode>(id, ctx.get(), groupbynode));
         } else if (auto samplenode =
                        std::dynamic_pointer_cast<const plan::RandomSampleNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: RandomSampleNode");
             operators.push_back(std::make_unique<PhyRandomSampleNode>(
                 id, ctx.get(), samplenode));
         } else if (auto rescoresnode =
                        std::dynamic_pointer_cast<const plan::RescoresNode>(
                            plannode)) {
+            tracer::AddEvent("create_operator: RescoresNode");
             operators.push_back(
                 std::make_unique<PhyRescoresNode>(id, ctx.get(), rescoresnode));
         }

internal/core/src/exec/Task.cpp

@@ -15,12 +15,13 @@
 // limitations under the License.
 
 #include "Task.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include <boost/lexical_cast.hpp>
 #include <boost/uuid/uuid_generators.hpp>
 #include <boost/uuid/uuid_io.hpp>
 #include "log/Log.h"
-
 namespace milvus {
 namespace exec {
 
@@ -146,6 +147,7 @@ Task::Next(ContinueFuture* future) {
     AssertInfo(state_ == TaskState::kRunning,
                "Task has already finished processing.");
 
+    tracer::AutoSpan span("Task::Next", tracer::GetRootSpan(), true);
     if (driver_factories_.empty()) {
         AssertInfo(
             consumer_supplier_ == nullptr,
@@ -199,10 +201,15 @@ Task::Next(ContinueFuture* future) {
             auto result = drivers_[i]->Next(blocking_state);
 
             if (result) {
+                tracer::AddEvent(
+                    fmt::format("driver_result_produced: driver_id={}, rows={}",
+                                i,
+                                result->childrens()[0]->size()));
                 return result;
             }
 
             if (blocking_state) {
+                tracer::AddEvent(fmt::format("driver_{}_blocked", i));
                 futures[i] = blocking_state->future();
             }
 
@@ -211,6 +218,10 @@ Task::Next(ContinueFuture* future) {
             }
         }
 
+        tracer::AddEvent(fmt::format("iteration: runnable={}, blocked={}",
+                                     runnable_drivers,
+                                     blocked_drivers));
+
         if (runnable_drivers == 0) {
             if (blocked_drivers > 0) {
                 if (!future) {

internal/core/src/exec/expression/BinaryArithOpEvalRangeExpr.cpp

@@ -21,6 +21,12 @@ namespace exec {
 
 void
 PhyBinaryArithOpEvalRangeExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyBinaryArithOpEvalRangeExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+    span.GetSpan()->SetAttribute("op_type", static_cast<int>(expr_->op_type_));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     switch (expr_->column_.data_type_) {

internal/core/src/exec/expression/BinaryRangeExpr.cpp

@@ -24,6 +24,11 @@ namespace exec {
 
 void
 PhyBinaryRangeFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyBinaryRangeFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     switch (expr_->column_.data_type_) {

internal/core/src/exec/expression/CallExpr.cpp

@@ -28,6 +28,9 @@ namespace exec {
 
 void
 PhyCallExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyCallExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("function_name", expr_->fun_name());
+
     auto offset_input = context.get_offset_input();
     SetHasOffsetInput(offset_input != nullptr);
     AssertInfo(inputs_.size() == expr_->inputs().size(),

internal/core/src/exec/expression/ColumnExpr.cpp

@@ -30,6 +30,9 @@ PhyColumnExpr::GetNextBatchSize() {
 
 void
 PhyColumnExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyColumnExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type", static_cast<int>(expr_->type()));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput(input != nullptr);
     switch (this->expr_->type()) {

internal/core/src/exec/expression/CompareExpr.cpp

@@ -15,9 +15,10 @@
 // limitations under the License.
 
 #include "CompareExpr.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 #include <optional>
 #include "query/Relational.h"
-
 namespace milvus {
 namespace exec {
 
@@ -214,6 +215,12 @@ PhyCompareFilterExpr::ExecCompareExprDispatcher(OpType op, EvalCtx& context) {
 
 void
 PhyCompareFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyCompareFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("op_type", static_cast<int>(expr_->op_type_));
+    span.GetSpan()->SetAttribute("left_indexed", is_left_indexed_);
+    span.GetSpan()->SetAttribute("right_indexed", is_right_indexed_);
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     // For segment both fields has no index, can use SIMD to speed up.

internal/core/src/exec/expression/ConjunctExpr.cpp

@@ -90,6 +90,10 @@ PhyConjunctFilterExpr::SkipFollowingExprs(int start) {
 
 void
 PhyConjunctFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyConjunctFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("is_and", is_and_);
+
     if (input_order_.empty()) {
         input_order_.resize(inputs_.size());
         for (size_t i = 0; i < inputs_.size(); i++) {

internal/core/src/exec/expression/ExistsExpr.cpp

@@ -26,6 +26,11 @@ namespace exec {
 
 void
 PhyExistsFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyExistsFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+
     context.set_apply_valid_data_after_flip(false);
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));

internal/core/src/exec/expression/Expr.cpp

@@ -17,6 +17,8 @@
 #include "Expr.h"
 
 #include "common/EasyAssert.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 #include "exec/expression/AlwaysTrueExpr.h"
 #include "exec/expression/BinaryArithOpEvalRangeExpr.h"
 #include "exec/expression/BinaryRangeExpr.h"
@@ -39,7 +41,6 @@
 #include "monitor/Monitor.h"
 
 #include <memory>
-
 namespace milvus {
 namespace exec {
 
@@ -49,6 +50,8 @@ ExprSet::Eval(int32_t begin,
               bool initialize,
               EvalCtx& context,
               std::vector<VectorPtr>& results) {
+    tracer::AutoSpan span("ExprSet::Eval", tracer::GetRootSpan(), true);
+
     results.resize(exprs_.size());
     for (size_t i = begin; i < end; ++i) {
         exprs_[i]->Eval(context, results[i]);

internal/core/src/exec/expression/JsonContainsExpr.cpp

@@ -24,6 +24,11 @@ namespace exec {
 
 void
 PhyJsonContainsFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyJsonContainsFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     if (expr_->vals_.empty()) {

internal/core/src/exec/expression/LogicalBinaryExpr.cpp

@@ -21,6 +21,8 @@ namespace exec {
 
 void
 PhyLogicalBinaryExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyLogicalBinaryExpr::Eval", tracer::GetRootSpan());
+
     AssertInfo(
         inputs_.size() == 2,
         "logical binary expr must have 2 inputs, but {} inputs are provided",

internal/core/src/exec/expression/LogicalUnaryExpr.cpp

@@ -21,6 +21,8 @@ namespace exec {
 
 void
 PhyLogicalUnaryExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyLogicalUnaryExpr::Eval", tracer::GetRootSpan());
+
     AssertInfo(inputs_.size() == 1,
                "logical unary expr must has one input, but now {}",
                inputs_.size());

internal/core/src/exec/expression/NullExpr.cpp

@@ -26,6 +26,10 @@ namespace exec {
 
 void
 PhyNullExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyNullExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+
     auto input = context.get_offset_input();
     switch (expr_->column_.data_type_) {
         case DataType::BOOL: {

internal/core/src/exec/expression/TermExpr.cpp

@@ -24,6 +24,11 @@ namespace exec {
 
 void
 PhyTermFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyTermFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     if (is_pk_field_ && !has_offset_input_) {

internal/core/src/exec/expression/UnaryExpr.cpp

@@ -155,6 +155,12 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplArrayForIndex<proto::plan::Array>(
 
 void
 PhyUnaryRangeFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span(
+        "PhyUnaryRangeFilterExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type",
+                                 static_cast<int>(expr_->column_.data_type_));
+    span.GetSpan()->SetAttribute("op_type", static_cast<int>(expr_->op_type_));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     switch (expr_->column_.data_type_) {

internal/core/src/exec/expression/ValueExpr.cpp

@@ -22,6 +22,9 @@ namespace exec {
 
 void
 PhyValueExpr::Eval(EvalCtx& context, VectorPtr& result) {
+    tracer::AutoSpan span("PhyValueExpr::Eval", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("data_type", static_cast<int>(expr_->type()));
+
     auto input = context.get_offset_input();
     SetHasOffsetInput((input != nullptr));
     int64_t real_batch_size = has_offset_input_

internal/core/src/exec/operator/CountNode.cpp

@@ -15,7 +15,8 @@
 // limitations under the License.
 
 #include "CountNode.h"
-
+#include "common/Tracer.h"
+#include "fmt/format.h"
 namespace milvus {
 namespace exec {
 
@@ -56,7 +57,7 @@ PhyCountNode::GetOutput() {
     if (is_finished_ || !no_more_input_) {
         return nullptr;
     }
-
+    tracer::AutoSpan span("PhyCountNode::Execute", tracer::GetRootSpan(), true);
     auto col_input = GetColumnVector(input_);
     TargetBitmapView view(col_input->GetRawData(), col_input->size());
     auto cnt = view.size() - view.count();
@@ -64,6 +65,7 @@ PhyCountNode::GetOutput() {
         std::move(*(wrap_num_entities(cnt, view.size()))));
     is_finished_ = true;
 
+    tracer::AddEvent(fmt::format("count_result: {}", cnt));
     return input_;
 }
 

internal/core/src/exec/operator/FilterBitsNode.cpp

@@ -15,6 +15,8 @@
 // limitations under the License.
 
 #include "FilterBitsNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include "monitor/Monitor.h"
 
@@ -63,6 +65,10 @@ PhyFilterBitsNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyFilterBitsNode::Execute", tracer::GetRootSpan(), true);
+    tracer::AddEvent(fmt::format("input_rows: {}", need_process_rows_));
+
     std::chrono::high_resolution_clock::time_point scalar_start =
         std::chrono::high_resolution_clock::now();
 
@@ -102,8 +108,10 @@ PhyFilterBitsNode::GetOutput() {
                bitset.size(),
                need_process_rows_);
     Assert(valid_bitset.size() == need_process_rows_);
+
+    auto filtered_count = bitset.count();
     auto filter_ratio =
-        bitset.size() != 0 ? 1 - float(bitset.count()) / bitset.size() : 0;
+        bitset.size() != 0 ? 1 - float(filtered_count) / bitset.size() : 0;
     milvus::monitor::internal_core_expr_filter_ratio.Observe(filter_ratio);
     // num_processed_rows_ = need_process_rows_;
     std::vector<VectorPtr> col_res;
@@ -117,6 +125,10 @@ PhyFilterBitsNode::GetOutput() {
     milvus::monitor::internal_core_search_latency_scalar.Observe(scalar_cost /
                                                                  1000);
 
+    tracer::AddEvent(fmt::format("output_rows: {}, filtered: {}",
+                                 need_process_rows_ - filtered_count,
+                                 filtered_count));
+
     return std::make_shared<RowVector>(col_res);
 }
 

internal/core/src/exec/operator/GroupByNode.cpp

@@ -15,10 +15,11 @@
 // limitations under the License.
 
 #include "GroupByNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include "exec/operator/groupby/SearchGroupByOperator.h"
 #include "monitor/Monitor.h"
-
 namespace milvus {
 namespace exec {
 
@@ -48,11 +49,16 @@ PhyGroupByNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyGroupByNode::Execute", tracer::GetRootSpan(), true);
+
     DeferLambda([&]() { is_finished_ = true; });
     if (input_ == nullptr) {
         return nullptr;
     }
 
+    tracer::AddEvent(fmt::format("group_size: {}", search_info_.group_size_));
+
     std::chrono::high_resolution_clock::time_point vector_start =
         std::chrono::high_resolution_clock::now();
 
@@ -81,6 +87,9 @@ PhyGroupByNode::GetOutput() {
                    search_result.group_by_values_.value().size(),
                    search_result.seg_offsets_.size());
     }
+    tracer::AddEvent(
+        fmt::format("grouped_results: {}", search_result.seg_offsets_.size()));
+
     query_context_->set_search_result(std::move(search_result));
     std::chrono::high_resolution_clock::time_point vector_end =
         std::chrono::high_resolution_clock::now();

internal/core/src/exec/operator/IterativeFilterNode.cpp

@@ -15,10 +15,11 @@
 // limitations under the License.
 
 #include "IterativeFilterNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include "exec/Driver.h"
 #include "monitor/Monitor.h"
-
 namespace milvus {
 namespace exec {
 PhyIterativeFilterNode::PhyIterativeFilterNode(
@@ -116,6 +117,9 @@ PhyIterativeFilterNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyIterativeFilterNode::Execute", tracer::GetRootSpan(), true);
+
     DeferLambda([&]() { is_finished_ = true; });
 
     if (input_ == nullptr) {
@@ -267,6 +271,12 @@ PhyIterativeFilterNode::GetOutput() {
     milvus::monitor::internal_core_search_latency_iterative_filter.Observe(
         scalar_cost / 1000);
 
+    if (!is_native_supported_) {
+        tracer::AddEvent(fmt::format("total_processed: {}, matched: {}",
+                                     need_process_rows_,
+                                     need_process_rows_ - bitset.count()));
+    }
+
     return input_;
 }
 

internal/core/src/exec/operator/MvccNode.cpp

@@ -15,7 +15,8 @@
 // limitations under the License.
 
 #include "MvccNode.h"
-
+#include "common/Tracer.h"
+#include "fmt/format.h"
 namespace milvus {
 namespace exec {
 
@@ -47,6 +48,8 @@ PhyMvccNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span("PhyMvccNode::Execute", tracer::GetRootSpan(), true);
+
     if (!is_source_node_ && input_ == nullptr) {
         return nullptr;
     }
@@ -55,6 +58,8 @@ PhyMvccNode::GetOutput() {
         is_finished_ = true;
         return nullptr;
     }
+
+    tracer::AddEvent(fmt::format("input_rows: {}", active_count_));
     // the first vector is filtering result and second bitset is a valid bitset
     // if valid_bitset[i]==false, means result[i] is null
     auto col_input = is_source_node_ ? std::make_shared<ColumnVector>(
@@ -69,6 +74,10 @@ PhyMvccNode::GetOutput() {
     segment_->mask_with_delete(data, active_count_, query_timestamp_);
     is_finished_ = true;
 
+    auto output_rows = active_count_ - data.count();
+    tracer::AddEvent(fmt::format(
+        "output_rows: {}, filtered: {}", output_rows, data.count()));
+
     // input_ have already been updated
     return std::make_shared<RowVector>(std::vector<VectorPtr>{col_input});
 }

internal/core/src/exec/operator/RandomSampleNode.cpp

@@ -15,10 +15,11 @@
 // limitations under the License.
 
 #include "RandomSampleNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include "exec/expression/Utils.h"
 #include "monitor/Monitor.h"
-
 namespace milvus {
 namespace exec {
 PhyRandomSampleNode::PhyRandomSampleNode(
@@ -93,6 +94,9 @@ PhyRandomSampleNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyRandomSampleNode::Execute", tracer::GetRootSpan(), true);
+
     if (!is_source_node_ && input_ == nullptr) {
         return nullptr;
     }
@@ -102,6 +106,9 @@ PhyRandomSampleNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AddEvent(fmt::format(
+        "sample_factor: {}, active_count: {}", factor_, active_count_));
+
     std::chrono::high_resolution_clock::time_point start =
         std::chrono::high_resolution_clock::now();
 
@@ -166,6 +173,17 @@ PhyRandomSampleNode::GetOutput() {
         std::chrono::duration<double, std::micro>(end - start).count();
     milvus::monitor::internal_core_search_latency_random_sample.Observe(
         duration / 1000);
+
+    if (result) {
+        auto result_col = GetColumnVector(result);
+        TargetBitmapView result_data(result_col->GetRawData(),
+                                     result_col->size());
+        auto sampled_count = result_col->size() - result_data.count();
+        tracer::AddEvent(fmt::format("sampled_count: {}, total_count: {}",
+                                     sampled_count,
+                                     active_count_));
+    }
+
     is_finished_ = true;
     return result;
 }

internal/core/src/exec/operator/RescoresNode.cpp

@@ -15,6 +15,8 @@
 // limitations under the License.
 
 #include "RescoresNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 #include <cstddef>
 #include "exec/operator/Utils.h"
 #include "log/Log.h"
@@ -52,6 +54,9 @@ PhyRescoresNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyRescoresNode::Execute", tracer::GetRootSpan(), true);
+
     DeferLambda([&]() { is_finished_ = true; });
 
     if (input_ == nullptr) {
@@ -180,6 +185,8 @@ PhyRescoresNode::GetOutput() {
             .count();
     milvus::monitor::internal_core_search_latency_rescore.Observe(scalar_cost /
                                                                   1000);
+
+    tracer::AddEvent(fmt::format("rescored_count: {}", offsets.size()));
     return input_;
 };
 

internal/core/src/exec/operator/VectorSearchNode.cpp

@@ -15,9 +15,10 @@
 // limitations under the License.
 
 #include "VectorSearchNode.h"
+#include "common/Tracer.h"
+#include "fmt/format.h"
 
 #include "monitor/Monitor.h"
-
 namespace milvus {
 namespace exec {
 
@@ -59,11 +60,17 @@ PhyVectorSearchNode::GetOutput() {
         return nullptr;
     }
 
+    tracer::AutoSpan span(
+        "PhyVectorSearchNode::Execute", tracer::GetRootSpan(), true);
+
     DeferLambda([&]() { is_finished_ = true; });
     if (input_ == nullptr) {
         return nullptr;
     }
 
+    span.GetSpan()->SetAttribute("search_type", search_info_.metric_type_);
+    span.GetSpan()->SetAttribute("topk", search_info_.topk_);
+
     std::chrono::high_resolution_clock::time_point vector_start =
         std::chrono::high_resolution_clock::now();
 
@@ -95,6 +102,10 @@ PhyVectorSearchNode::GetOutput() {
                             search_result);
 
     search_result.total_data_cnt_ = final_view.size();
+
+    span.GetSpan()->SetAttribute(
+        "result_count", static_cast<int>(search_result.seg_offsets_.size()));
+
     query_context_->set_search_result(std::move(search_result));
     std::chrono::high_resolution_clock::time_point vector_end =
         std::chrono::high_resolution_clock::now();

internal/core/src/index/BitmapIndex.cpp

@@ -594,6 +594,8 @@ BitmapIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
 template <typename T>
 const TargetBitmap
 BitmapIndex<T>::In(const size_t n, const T* values) {
+    tracer::AutoSpan span("BitmapIndex::In", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
 
@@ -633,6 +635,8 @@ BitmapIndex<T>::In(const size_t n, const T* values) {
 template <typename T>
 const TargetBitmap
 BitmapIndex<T>::NotIn(const size_t n, const T* values) {
+    tracer::AutoSpan span("BitmapIndex::NotIn", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
 
     if (is_mmap_) {
@@ -682,6 +686,8 @@ BitmapIndex<T>::NotIn(const size_t n, const T* values) {
 template <typename T>
 const TargetBitmap
 BitmapIndex<T>::IsNull() {
+    tracer::AutoSpan span("BitmapIndex::IsNull", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, true);
     res &= valid_bitset_;
@@ -692,6 +698,8 @@ BitmapIndex<T>::IsNull() {
 template <typename T>
 TargetBitmap
 BitmapIndex<T>::IsNotNull() {
+    tracer::AutoSpan span("BitmapIndex::IsNotNull", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, true);
     res &= valid_bitset_;
@@ -701,6 +709,8 @@ BitmapIndex<T>::IsNotNull() {
 template <typename T>
 TargetBitmap
 BitmapIndex<T>::RangeForBitset(const T value, const OpType op) {
+    tracer::AutoSpan span("BitmapIndex::RangeForBitset", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (ShouldSkip(value, value, op)) {
@@ -773,6 +783,8 @@ BitmapIndex<T>::Range(const T value, OpType op) {
 template <typename T>
 TargetBitmap
 BitmapIndex<T>::RangeForMmap(const T value, const OpType op) {
+    tracer::AutoSpan span("BitmapIndex::RangeForMmap", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (ShouldSkip(value, value, op)) {
@@ -835,6 +847,9 @@ BitmapIndex<T>::RangeForMmap(const T value, const OpType op) {
 template <typename T>
 TargetBitmap
 BitmapIndex<T>::RangeForRoaring(const T value, const OpType op) {
+    tracer::AutoSpan span("BitmapIndex::RangeForRoaring",
+                          tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (ShouldSkip(value, value, op)) {
@@ -899,6 +914,8 @@ BitmapIndex<T>::RangeForBitset(const T lower_value,
                                bool lb_inclusive,
                                const T upper_value,
                                bool ub_inclusive) {
+    tracer::AutoSpan span("BitmapIndex::RangeForBitset", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (lower_value > upper_value ||
@@ -975,6 +992,8 @@ BitmapIndex<T>::RangeForMmap(const T lower_value,
                              bool lb_inclusive,
                              const T upper_value,
                              bool ub_inclusive) {
+    tracer::AutoSpan span("BitmapIndex::RangeForMmap", tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (lower_value > upper_value ||
@@ -1034,6 +1053,9 @@ BitmapIndex<T>::RangeForRoaring(const T lower_value,
                                 bool lb_inclusive,
                                 const T upper_value,
                                 bool ub_inclusive) {
+    tracer::AutoSpan span("BitmapIndex::RangeForRoaring",
+                          tracer::GetRootSpan());
+
     AssertInfo(is_built_, "index has not been built");
     TargetBitmap res(total_num_rows_, false);
     if (lower_value > upper_value ||
@@ -1107,6 +1129,7 @@ std::optional<T>
 BitmapIndex<T>::Reverse_Lookup(size_t idx) const {
     AssertInfo(is_built_, "index has not been built");
     AssertInfo(idx < total_num_rows_, "out of range of total count");
+    tracer::AutoSpan span("BitmapIndex::Reverse_Lookup", tracer::GetRootSpan());
 
     if (!valid_bitset_[idx]) {
         return std::nullopt;
@@ -1225,6 +1248,7 @@ template <>
 const TargetBitmap
 BitmapIndex<std::string>::Query(const DatasetPtr& dataset) {
     AssertInfo(is_built_, "index has not been built");
+    tracer::AutoSpan span("BitmapIndex::Query", tracer::GetRootSpan());
 
     auto op = dataset->Get<OpType>(OPERATOR_TYPE);
     auto val = dataset->Get<std::string>(MATCH_VALUE);
@@ -1272,6 +1296,8 @@ template <>
 const TargetBitmap
 BitmapIndex<std::string>::RegexQuery(const std::string& regex_pattern) {
     AssertInfo(is_built_, "index has not been built");
+    tracer::AutoSpan span("BitmapIndex::RegexQuery", tracer::GetRootSpan());
+
     RegexMatcher matcher(regex_pattern);
     TargetBitmap res(total_num_rows_, false);
     if (is_mmap_) {

internal/core/src/index/InvertedIndexTantivy.cpp

@@ -12,6 +12,7 @@
 #include "tantivy-binding.h"
 #include "common/Slice.h"
 #include "common/RegexQuery.h"
+#include "common/Tracer.h"
 #include "storage/LocalChunkManagerSingleton.h"
 #include "index/InvertedIndexTantivy.h"
 #include "index/InvertedIndexUtil.h"
@@ -274,6 +275,7 @@ InvertedIndexTantivy<T>::RetainTantivyIndexFiles(
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::In(size_t n, const T* values) {
+    tracer::AutoSpan span("InvertedIndexTantivy::In", tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     wrapper_->terms_query(values, n, &bitset);
     return bitset;
@@ -282,6 +284,8 @@ InvertedIndexTantivy<T>::In(size_t n, const T* values) {
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::IsNull() {
+    tracer::AutoSpan span("InvertedIndexTantivy::IsNull",
+                          tracer::GetRootSpan());
     int64_t count = Count();
     TargetBitmap bitset(count);
 
@@ -306,6 +310,8 @@ InvertedIndexTantivy<T>::IsNull() {
 template <typename T>
 TargetBitmap
 InvertedIndexTantivy<T>::IsNotNull() {
+    tracer::AutoSpan span("InvertedIndexTantivy::IsNotNull",
+                          tracer::GetRootSpan());
     int64_t count = Count();
     TargetBitmap bitset(count, true);
 
@@ -331,6 +337,8 @@ template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::InApplyFilter(
     size_t n, const T* values, const std::function<bool(size_t)>& filter) {
+    tracer::AutoSpan span("InvertedIndexTantivy::InApplyFilter",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     wrapper_->terms_query(values, n, &bitset);
     // todo(SpadeA): could push-down the filter to tantivy query
@@ -342,6 +350,8 @@ template <typename T>
 void
 InvertedIndexTantivy<T>::InApplyCallback(
     size_t n, const T* values, const std::function<void(size_t)>& callback) {
+    tracer::AutoSpan span("InvertedIndexTantivy::InApplyCallback",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     wrapper_->terms_query(values, n, &bitset);
     // todo(SpadeA): could push-down the callback to tantivy query
@@ -351,6 +361,7 @@ InvertedIndexTantivy<T>::InApplyCallback(
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::NotIn(size_t n, const T* values) {
+    tracer::AutoSpan span("InvertedIndexTantivy::NotIn", tracer::GetRootSpan());
     int64_t count = Count();
     TargetBitmap bitset(count);
     wrapper_->terms_query(values, n, &bitset);
@@ -378,6 +389,7 @@ InvertedIndexTantivy<T>::NotIn(size_t n, const T* values) {
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::Range(T value, OpType op) {
+    tracer::AutoSpan span("InvertedIndexTantivy::Range", tracer::GetRootSpan());
     TargetBitmap bitset(Count());
 
     switch (op) {
@@ -407,6 +419,8 @@ InvertedIndexTantivy<T>::Range(T lower_bound_value,
                                bool lb_inclusive,
                                T upper_bound_value,
                                bool ub_inclusive) {
+    tracer::AutoSpan span("InvertedIndexTantivy::RangeWithBounds",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     wrapper_->range_query(lower_bound_value,
                           upper_bound_value,
@@ -419,6 +433,8 @@ InvertedIndexTantivy<T>::Range(T lower_bound_value,
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::PrefixMatch(const std::string_view prefix) {
+    tracer::AutoSpan span("InvertedIndexTantivy::PrefixMatch",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     std::string s(prefix);
     wrapper_->prefix_query(s, &bitset);
@@ -434,6 +450,7 @@ InvertedIndexTantivy<T>::Query(const DatasetPtr& dataset) {
 template <>
 const TargetBitmap
 InvertedIndexTantivy<std::string>::Query(const DatasetPtr& dataset) {
+    tracer::AutoSpan span("InvertedIndexTantivy::Query", tracer::GetRootSpan());
     auto op = dataset->Get<OpType>(OPERATOR_TYPE);
     if (op == OpType::PrefixMatch) {
         auto prefix = dataset->Get<std::string>(MATCH_VALUE);
@@ -445,6 +462,8 @@ InvertedIndexTantivy<std::string>::Query(const DatasetPtr& dataset) {
 template <typename T>
 const TargetBitmap
 InvertedIndexTantivy<T>::RegexQuery(const std::string& regex_pattern) {
+    tracer::AutoSpan span("InvertedIndexTantivy::RegexQuery",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(Count());
     wrapper_->regex_query(regex_pattern, &bitset);
     return bitset;

internal/core/src/index/JsonFlatIndex.h

@@ -38,6 +38,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     const TargetBitmap
     In(size_t n, const T* values) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::In",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         for (size_t i = 0; i < n; ++i) {
             this->wrapper_->json_term_query(json_path_, values[i], &bitset);
@@ -47,6 +49,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     const TargetBitmap
     IsNull() override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::IsNull",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         this->wrapper_->json_exist_query(json_path_, &bitset);
         bitset.flip();
@@ -55,6 +59,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     TargetBitmap
     IsNotNull() override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::IsNotNull",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         this->wrapper_->json_exist_query(json_path_, &bitset);
         return bitset;
@@ -65,6 +71,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
         size_t n,
         const T* values,
         const std::function<bool(size_t /* offset */)>& filter) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::InApplyFilter",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         for (size_t i = 0; i < n; ++i) {
             this->wrapper_->json_term_query(json_path_, values[i], &bitset);
@@ -78,6 +86,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
         size_t n,
         const T* values,
         const std::function<void(size_t /* offset */)>& callback) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::InApplyCallback",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         for (size_t i = 0; i < n; ++i) {
             this->wrapper_->json_term_query(json_path_, values[i], &bitset);
@@ -87,6 +97,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     const TargetBitmap
     NotIn(size_t n, const T* values) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::NotIn",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         for (size_t i = 0; i < n; ++i) {
             this->wrapper_->json_term_query(json_path_, values[i], &bitset);
@@ -104,6 +116,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
     const TargetBitmap
     Range(T value, OpType op) override {
         LOG_INFO("[executor] JsonFlatIndexQueryExecutor Range");
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::Range",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         switch (op) {
             case OpType::LessThan: {
@@ -139,6 +153,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
           bool lb_inclusive,
           T upper_bound_value,
           bool ub_inclusive) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::RangeWithBounds",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         this->wrapper_->json_range_query(json_path_,
                                          lower_bound_value,
@@ -153,6 +169,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     const TargetBitmap
     PrefixMatch(const std::string_view prefix) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::PrefixMatch",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         this->wrapper_->json_prefix_query(
             json_path_, std::string(prefix), &bitset);
@@ -161,6 +179,8 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
 
     const TargetBitmap
     RegexQuery(const std::string& pattern) override {
+        tracer::AutoSpan span("JsonFlatIndexQueryExecutor::RegexQuery",
+                              tracer::GetRootSpan());
         TargetBitmap bitset(this->Count());
         this->wrapper_->json_regex_query(json_path_, pattern, &bitset);
         return bitset;

internal/core/src/index/NgramInvertedIndex.cpp

@@ -178,6 +178,8 @@ std::optional<TargetBitmap>
 NgramInvertedIndex::ExecuteQuery(const std::string& literal,
                                  proto::plan::OpType op_type,
                                  exec::SegmentExpr* segment) {
+    tracer::AutoSpan span(
+        "NgramInvertedIndex::ExecuteQuery", tracer::GetRootSpan(), true);
     if (literal.length() < min_gram_) {
         return std::nullopt;
     }
@@ -186,6 +188,11 @@ NgramInvertedIndex::ExecuteQuery(const std::string& literal,
         case proto::plan::OpType::Match:
             return MatchQuery(literal, segment);
         case proto::plan::OpType::InnerMatch: {
+            span.GetSpan()->SetAttribute("op_type", "InnerMatch");
+            span.GetSpan()->SetAttribute("query_literal_length",
+                                         static_cast<int>(literal.length()));
+            span.GetSpan()->SetAttribute("min_gram", min_gram_);
+            span.GetSpan()->SetAttribute("max_gram", max_gram_);
             bool need_post_filter = literal.length() > max_gram_;
 
             if (schema_.data_type() == proto::schema::DataType::JSON) {
@@ -211,6 +218,11 @@ NgramInvertedIndex::ExecuteQuery(const std::string& literal,
             }
         }
         case proto::plan::OpType::PrefixMatch: {
+            span.GetSpan()->SetAttribute("op_type", "PrefixMatch");
+            span.GetSpan()->SetAttribute("query_literal_length",
+                                         static_cast<int>(literal.length()));
+            span.GetSpan()->SetAttribute("min_gram", min_gram_);
+            span.GetSpan()->SetAttribute("max_gram", max_gram_);
             if (schema_.data_type() == proto::schema::DataType::JSON) {
                 auto predicate = [&literal, this](const milvus::Json& data) {
                     auto x =
@@ -239,6 +251,11 @@ NgramInvertedIndex::ExecuteQuery(const std::string& literal,
             }
         }
         case proto::plan::OpType::PostfixMatch: {
+            span.GetSpan()->SetAttribute("op_type", "PostfixMatch");
+            span.GetSpan()->SetAttribute("query_literal_length",
+                                         static_cast<int>(literal.length()));
+            span.GetSpan()->SetAttribute("min_gram", min_gram_);
+            span.GetSpan()->SetAttribute("max_gram", max_gram_);
             if (schema_.data_type() == proto::schema::DataType::JSON) {
                 auto predicate = [&literal, this](const milvus::Json& data) {
                     auto x =
@@ -301,6 +318,9 @@ NgramInvertedIndex::ExecuteQueryWithPredicate(
     TargetBitmap bitset{static_cast<size_t>(Count())};
     wrapper_->ngram_match_query(literal, min_gram_, max_gram_, &bitset);
 
+    auto ngram_hit_count = bitset.count();
+    auto final_result_count = ngram_hit_count;
+
     if (need_post_filter) {
         TargetBitmapView res(bitset);
         TargetBitmap valid(res.size(), true);
@@ -321,6 +341,16 @@ NgramInvertedIndex::ExecuteQueryWithPredicate(
 
         segment->ProcessAllDataChunk<T>(
             execute_batch, std::nullptr_t{}, res, valid_res);
+        final_result_count = bitset.count();
+    }
+
+    if (auto root_span = tracer::GetRootSpan()) {
+        root_span->SetAttribute("need_post_filter", need_post_filter);
+        root_span->SetAttribute("ngram_hit_count",
+                                static_cast<int>(ngram_hit_count));
+        root_span->SetAttribute("final_result_count",
+                                static_cast<int>(final_result_count));
+        root_span->SetAttribute("total_count", static_cast<int>(bitset.size()));
     }
 
     return std::optional<TargetBitmap>(std::move(bitset));
@@ -359,6 +389,12 @@ split_by_wildcard(const std::string& literal) {
 std::optional<TargetBitmap>
 NgramInvertedIndex::MatchQuery(const std::string& literal,
                                exec::SegmentExpr* segment) {
+    if (auto root_span = tracer::GetRootSpan()) {
+        root_span->SetAttribute("match_query_literal_length",
+                                static_cast<int>(literal.length()));
+        root_span->SetAttribute("match_query_min_gram", min_gram_);
+        root_span->SetAttribute("match_query_max_gram", max_gram_);
+    }
     TargetBitmap bitset{static_cast<size_t>(Count())};
     auto literals = split_by_wildcard(literal);
     for (const auto& l : literals) {
@@ -376,6 +412,8 @@ NgramInvertedIndex::MatchQuery(const std::string& literal,
     auto regex_pattern = translator(literal);
     RegexMatcher matcher(regex_pattern);
 
+    auto ngram_hit_count = bitset.count();
+
     if (schema_.data_type() == proto::schema::DataType::JSON) {
         auto predicate = [&literal, &matcher, this](const milvus::Json& data) {
             auto x = data.template at<std::string_view>(this->nested_path_);
@@ -423,6 +461,17 @@ NgramInvertedIndex::MatchQuery(const std::string& literal,
             execute_batch, std::nullptr_t{}, res, valid_res);
     }
 
+    auto final_result_count = bitset.count();
+
+    if (auto root_span = tracer::GetRootSpan()) {
+        root_span->SetAttribute("match_ngram_hit_count",
+                                static_cast<int>(ngram_hit_count));
+        root_span->SetAttribute("match_final_result_count",
+                                static_cast<int>(final_result_count));
+        root_span->SetAttribute("match_total_count",
+                                static_cast<int>(bitset.size()));
+    }
+
     return std::optional<TargetBitmap>(std::move(bitset));
 }
 

internal/core/src/index/StringIndexMarisa.cpp

@@ -252,6 +252,7 @@ StringIndexMarisa::Load(milvus::tracer::TraceContext ctx,
 
 const TargetBitmap
 StringIndexMarisa::In(size_t n, const std::string* values) {
+    tracer::AutoSpan span("StringIndexMarisa::In", tracer::GetRootSpan());
     TargetBitmap bitset(str_ids_.size());
     for (size_t i = 0; i < n; i++) {
         auto str = values[i];
@@ -268,6 +269,7 @@ StringIndexMarisa::In(size_t n, const std::string* values) {
 
 const TargetBitmap
 StringIndexMarisa::NotIn(size_t n, const std::string* values) {
+    tracer::AutoSpan span("StringIndexMarisa::NotIn", tracer::GetRootSpan());
     TargetBitmap bitset(str_ids_.size(), true);
     for (size_t i = 0; i < n; i++) {
         auto str = values[i];
@@ -286,6 +288,7 @@ StringIndexMarisa::NotIn(size_t n, const std::string* values) {
 
 const TargetBitmap
 StringIndexMarisa::IsNull() {
+    tracer::AutoSpan span("StringIndexMarisa::IsNull", tracer::GetRootSpan());
     TargetBitmap bitset(str_ids_.size());
     SetNull(bitset);
     return bitset;
@@ -293,6 +296,7 @@ StringIndexMarisa::IsNull() {
 
 void
 StringIndexMarisa::SetNull(TargetBitmap& bitset) {
+    tracer::AutoSpan span("StringIndexMarisa::SetNull", tracer::GetRootSpan());
     for (size_t i = 0; i < bitset.size(); i++) {
         if (str_ids_[i] == MARISA_NULL_KEY_ID) {
             bitset.set(i);
@@ -302,6 +306,8 @@ StringIndexMarisa::SetNull(TargetBitmap& bitset) {
 
 void
 StringIndexMarisa::ResetNull(TargetBitmap& bitset) {
+    tracer::AutoSpan span("StringIndexMarisa::ResetNull",
+                          tracer::GetRootSpan());
     for (size_t i = 0; i < bitset.size(); i++) {
         if (str_ids_[i] == MARISA_NULL_KEY_ID) {
             bitset.reset(i);
@@ -311,6 +317,8 @@ StringIndexMarisa::ResetNull(TargetBitmap& bitset) {
 
 TargetBitmap
 StringIndexMarisa::IsNotNull() {
+    tracer::AutoSpan span("StringIndexMarisa::IsNotNull",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(str_ids_.size());
     for (size_t i = 0; i < bitset.size(); i++) {
         if (str_ids_[i] != MARISA_NULL_KEY_ID) {
@@ -322,6 +330,7 @@ StringIndexMarisa::IsNotNull() {
 
 const TargetBitmap
 StringIndexMarisa::Range(std::string value, OpType op) {
+    tracer::AutoSpan span("StringIndexMarisa::Range", tracer::GetRootSpan());
     auto count = Count();
     TargetBitmap bitset(count);
     std::vector<size_t> ids;
@@ -444,6 +453,7 @@ StringIndexMarisa::Range(std::string lower_bound_value,
                          bool lb_inclusive,
                          std::string upper_bound_value,
                          bool ub_inclusive) {
+    tracer::AutoSpan span("StringIndexMarisa::Range", tracer::GetRootSpan());
     auto count = Count();
     TargetBitmap bitset(count);
     if (lower_bound_value.compare(upper_bound_value) > 0 ||
@@ -495,6 +505,8 @@ StringIndexMarisa::Range(std::string lower_bound_value,
 
 const TargetBitmap
 StringIndexMarisa::PrefixMatch(std::string_view prefix) {
+    tracer::AutoSpan span("StringIndexMarisa::PrefixMatch",
+                          tracer::GetRootSpan());
     TargetBitmap bitset(str_ids_.size());
     auto matched = prefix_match(prefix);
     for (const auto str_id : matched) {
@@ -547,6 +559,8 @@ StringIndexMarisa::lookup(const std::string_view str) {
 
 std::vector<size_t>
 StringIndexMarisa::prefix_match(const std::string_view prefix) {
+    tracer::AutoSpan span("StringIndexMarisa::prefix_match",
+                          tracer::GetRootSpan());
     std::vector<size_t> ret;
     marisa::Agent agent;
     agent.set_query(prefix.data());
@@ -557,6 +571,8 @@ StringIndexMarisa::prefix_match(const std::string_view prefix) {
 }
 std::optional<std::string>
 StringIndexMarisa::Reverse_Lookup(size_t offset) const {
+    tracer::AutoSpan span("StringIndexMarisa::Reverse_Lookup",
+                          tracer::GetRootSpan());
     AssertInfo(offset < str_ids_.size(), "out of range of total count");
     marisa::Agent agent;
     if (str_ids_[offset] < 0) {

internal/core/src/index/TextMatchIndex.cpp

@@ -303,6 +303,7 @@ TextMatchIndex::RegisterTokenizer(const char* tokenizer_name,
 
 TargetBitmap
 TextMatchIndex::MatchQuery(const std::string& query) {
+    tracer::AutoSpan span("TextMatchIndex::MatchQuery", tracer::GetRootSpan());
     if (shouldTriggerCommit()) {
         Commit();
         Reload();
@@ -318,6 +319,8 @@ TextMatchIndex::MatchQuery(const std::string& query) {
 
 TargetBitmap
 TextMatchIndex::PhraseMatchQuery(const std::string& query, uint32_t slop) {
+    tracer::AutoSpan span("TextMatchIndex::PhraseMatchQuery",
+                          tracer::GetRootSpan());
     if (shouldTriggerCommit()) {
         Commit();
         Reload();

internal/core/src/query/ExecPlanNodeVisitor.cpp

@@ -40,6 +40,10 @@ BitsetType
 ExecPlanNodeVisitor::ExecuteTask(
     plan::PlanFragment& plan,
     std::shared_ptr<milvus::exec::QueryContext> query_context) {
+    tracer::AutoSpan span("ExecuteTask", tracer::GetRootSpan(), true);
+    span.GetSpan()->SetAttribute("active_count",
+                                 query_context->get_active_count());
+
     LOG_DEBUG("plannode: {}, active_count: {}, timestamp: {}",
               plan.plan_node_->ToString(),
               query_context->get_active_count(),
@@ -66,6 +70,10 @@ ExecPlanNodeVisitor::ExecuteTask(
             ThrowInfo(UnexpectedError, "expr return type not matched");
         }
     }
+
+    span.GetSpan()->SetAttribute("total_rows", processed_num);
+    span.GetSpan()->SetAttribute("matched_rows", bitset_holder.count());
+
     return bitset_holder;
 }
 
@@ -180,7 +188,7 @@ ExecPlanNodeVisitor::visit(RetrievePlanNode& node) {
             op_context.storage_usage.scanned_total_bytes.load();
     } else {
         retrieve_result.total_data_cnt_ = bitset_holder.size();
-        tracer::AutoSpan _("Find Limit Pk", tracer::GetRootSpan());
+        tracer::AutoSpan _("Find Limit Pk", tracer::GetRootSpan(), true);
         auto results_pair = segment->find_first(node.limit_, bitset_holder);
         retrieve_result.result_offsets_ = std::move(results_pair.first);
         retrieve_result.has_more_result = results_pair.second;

internal/core/src/segcore/SegmentInterface.cpp

@@ -117,7 +117,7 @@ SegmentInternalInterface::Retrieve(tracer::TraceContext* trace_ctx,
                                    int32_t consistency_level,
                                    Timestamp collection_ttl) const {
     std::shared_lock lck(mutex_);
-    tracer::AutoSpan span("Retrieve", tracer::GetRootSpan());
+    tracer::AutoSpan span("Retrieve", tracer::GetRootSpan(), true);
     auto results = std::make_unique<proto::segcore::RetrieveResults>();
     query::ExecPlanNodeVisitor visitor(
         *this, timestamp, consistency_level, collection_ttl);