From ea307ea3c9571e172a0a10e51b28ee3b90c253a3 Mon Sep 17 00:00:00 2001 From: congqixia Date: Wed, 24 Sep 2025 10:12:03 +0800 Subject: [PATCH] fix: [StorageV2] Make DiskFileManager use fs from context (#44535) Related to #44534 Datanode shall not use singleton fs after 2.6+. This patch make disk file manager use filesystem passed by fileManagerContext instead of errorous singleton one. --------- Signed-off-by: Congqi Xia --- .../core/src/index/json_stats/JsonKeyStats.cpp | 9 +++++++-- .../core/src/storage/DiskFileManagerImpl.cpp | 16 ++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/core/src/index/json_stats/JsonKeyStats.cpp b/internal/core/src/index/json_stats/JsonKeyStats.cpp index 7f23ba4e39..ddb9d6453e 100644 --- a/internal/core/src/index/json_stats/JsonKeyStats.cpp +++ b/internal/core/src/index/json_stats/JsonKeyStats.cpp @@ -22,6 +22,7 @@ #include "index/json_stats/bson_builder.h" #include "index/InvertedIndexUtil.h" #include "index/Utils.h" +#include "milvus-storage/filesystem/fs.h" #include "storage/MmapManager.h" #include "storage/Util.h" #include "common/bson_view.h" @@ -76,8 +77,12 @@ JsonKeyStats::JsonKeyStats(const storage::FileManagerContext& ctx, // TODO: add params to modify batch size and max file size auto conf = milvus_storage::StorageConfig(); conf.part_size = DEFAULT_PART_UPLOAD_SIZE; - auto trueFs = milvus_storage::ArrowFileSystemSingleton::GetInstance() - .GetArrowFileSystem(); + auto trueFs = ctx.fs; + // try singleton if possible + if (!trueFs) { + trueFs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + } if (!trueFs) { ThrowInfo(ErrorCode::UnexpectedError, "Failed to get filesystem"); } diff --git a/internal/core/src/storage/DiskFileManagerImpl.cpp b/internal/core/src/storage/DiskFileManagerImpl.cpp index 7d6c3098f3..9372717c31 100644 --- a/internal/core/src/storage/DiskFileManagerImpl.cpp +++ b/internal/core/src/storage/DiskFileManagerImpl.cpp @@ -171,9 +171,11 @@ DiskFileManagerImpl::OpenInputStream(const std::string& filename) { auto local_file_name = GetFileName(filename); auto remote_file_path = GetRemoteIndexPathV2(local_file_name); - auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() - .GetArrowFileSystem(); - + auto fs = fs_; + if (!fs) { + fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + } auto remote_file = fs->OpenInputFile(remote_file_path); AssertInfo(remote_file.ok(), "failed to open remote file"); return std::static_pointer_cast( @@ -186,9 +188,11 @@ DiskFileManagerImpl::OpenOutputStream(const std::string& filename) { auto local_file_name = GetFileName(filename); auto remote_file_path = GetRemoteIndexPathV2(local_file_name); - auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() - .GetArrowFileSystem(); - + auto fs = fs_; + if (!fs) { + fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + } auto remote_stream = fs->OpenOutputStream(remote_file_path); AssertInfo(remote_stream.ok(), "failed to open remote stream, reason: {}",