diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp
index 9ec211936a..8f4ecad336 100644
--- a/internal/core/src/segcore/SegmentSealedImpl.cpp
+++ b/internal/core/src/segcore/SegmentSealedImpl.cpp
@@ -182,9 +182,12 @@ SegmentSealedImpl::LoadScalarIndex(const LoadIndexInfo& info) {
 
     set_bit(index_ready_bitset_, field_id, true);
     update_row_count(row_count);
-    // release field column
-    fields_.erase(field_id);
-    set_bit(field_data_ready_bitset_, field_id, false);
+    // release field column if the index contains raw data
+    if (scalar_indexings_[field_id]->HasRawData() &&
+        get_bit(field_data_ready_bitset_, field_id)) {
+        fields_.erase(field_id);
+        set_bit(field_data_ready_bitset_, field_id, false);
+    }
 
     lck.unlock();
 }