diff --git a/internal/core/src/index/InvertedIndexTantivy.cpp b/internal/core/src/index/InvertedIndexTantivy.cpp
index b2c4236514..c5ae8a81ad 100644
--- a/internal/core/src/index/InvertedIndexTantivy.cpp
+++ b/internal/core/src/index/InvertedIndexTantivy.cpp
@@ -148,6 +148,11 @@ InvertedIndexTantivy<T>::Upload(const Config& config) {
     }
     auto binary_set = Serialize(config);
     mem_file_manager_->AddFile(binary_set);
+    auto remote_mem_path_to_size =
+        mem_file_manager_->GetRemotePathsToFileSize();
+    for (auto& file : remote_mem_path_to_size) {
+        ret.Append(file.first, nullptr, file.second);
+    }
     return ret;
 }
 
@@ -181,28 +186,34 @@ InvertedIndexTantivy<T>::Load(milvus::tracer::TraceContext ctx,
                                          return file == index_type_file;
                                      }),
                       files_value.end());
-    disk_file_manager_->CacheIndexToDisk(files_value);
-    wrapper_ = std::make_shared<TantivyIndexWrapper>(prefix.c_str());
+
     auto index_valid_data_file =
         mem_file_manager_->GetRemoteIndexObjectPrefix() +
         std::string("/index_null_offset");
-    std::vector<std::string> file;
-    file.push_back(index_valid_data_file);
-    auto index_datas = mem_file_manager_->LoadIndexToMemory(file);
-    AssembleIndexDatas(index_datas);
-    BinarySet binary_set;
-    for (auto& [key, data] : index_datas) {
-        auto size = data->DataSize();
-        auto deleter = [&](uint8_t*) {};  // avoid repeated deconstruction
-        auto buf = std::shared_ptr<uint8_t[]>(
-            (uint8_t*)const_cast<void*>(data->Data()), deleter);
-        binary_set.Append(key, buf, size);
+    auto it = std::find(
+        files_value.begin(), files_value.end(), index_valid_data_file);
+    if (it != files_value.end()) {
+        files_value.erase(it);
+        std::vector<std::string> file;
+        file.push_back(index_valid_data_file);
+        auto index_datas = mem_file_manager_->LoadIndexToMemory(file);
+        AssembleIndexDatas(index_datas);
+        BinarySet binary_set;
+        for (auto& [key, data] : index_datas) {
+            auto size = data->DataSize();
+            auto deleter = [&](uint8_t*) {};  // avoid repeated deconstruction
+            auto buf = std::shared_ptr<uint8_t[]>(
+                (uint8_t*)const_cast<void*>(data->Data()), deleter);
+            binary_set.Append(key, buf, size);
+        }
+        auto index_valid_data = binary_set.GetByName("index_null_offset");
+        null_offset.resize((size_t)index_valid_data->size / sizeof(size_t));
+        memcpy(null_offset.data(),
+               index_valid_data->data.get(),
+               (size_t)index_valid_data->size);
     }
-    auto index_valid_data = binary_set.GetByName("index_null_offset");
-    null_offset.resize((size_t)index_valid_data->size / sizeof(size_t));
-    memcpy(null_offset.data(),
-           index_valid_data->data.get(),
-           (size_t)index_valid_data->size);
+    disk_file_manager_->CacheIndexToDisk(files_value);
+    wrapper_ = std::make_shared<TantivyIndexWrapper>(prefix.c_str());
 }
 
 inline void