diff --git a/internal/core/src/segcore/DeletedRecord.h b/internal/core/src/segcore/DeletedRecord.h
index 507ec5435f..fddb49d63a 100644
--- a/internal/core/src/segcore/DeletedRecord.h
+++ b/internal/core/src/segcore/DeletedRecord.h
@@ -67,8 +67,11 @@ class DeletedRecord {
 
     ~DeletedRecord() {
         if constexpr (is_sealed) {
-            cachinglayer::Manager::GetInstance().RefundLoadedResource(
-                {estimated_memory_size_, 0});
+            if (estimated_memory_size_ > 0) {
+                cachinglayer::Manager::GetInstance().RefundLoadedResource(
+                    {estimated_memory_size_, 0});
+                estimated_memory_size_ = 0;
+            }
         }
     }
 
diff --git a/internal/core/src/segcore/InsertRecord.h b/internal/core/src/segcore/InsertRecord.h
index 9303a5a4a5..6aa3322d05 100644
--- a/internal/core/src/segcore/InsertRecord.h
+++ b/internal/core/src/segcore/InsertRecord.h
@@ -461,6 +461,14 @@ class InsertRecordSealed {
         }
     }
 
+    ~InsertRecordSealed() {
+        if (estimated_memory_size_ > 0) {
+            cachinglayer::Manager::GetInstance().RefundLoadedResource(
+                {static_cast<int64_t>(estimated_memory_size_), 0});
+            estimated_memory_size_ = 0;
+        }
+    }
+
     bool
     contain(const PkType& pk) const {
         return pk2offset_->contain(pk);
@@ -587,9 +595,11 @@ class InsertRecordSealed {
         timestamp_index_ = TimestampIndex();
         pk2offset_->clear();
         reserved = 0;
-        cachinglayer::Manager::GetInstance().RefundLoadedResource(
-            {static_cast<int64_t>(estimated_memory_size_), 0});
-        estimated_memory_size_ = 0;
+        if (estimated_memory_size_ > 0) {
+            cachinglayer::Manager::GetInstance().RefundLoadedResource(
+                {static_cast<int64_t>(estimated_memory_size_), 0});
+            estimated_memory_size_ = 0;
+        }
     }
 
  public: