diff --git a/internal/core/src/clustering/KmeansClusteringTest.cpp b/internal/core/src/clustering/KmeansClusteringTest.cpp index 55646686a3..a47ac51d40 100644 --- a/internal/core/src/clustering/KmeansClusteringTest.cpp +++ b/internal/core/src/clustering/KmeansClusteringTest.cpp @@ -189,6 +189,7 @@ test_run() { std::string root_path = "/tmp/test-kmeans-clustering/"; auto storage_config = gen_local_storage_config(root_path); auto cm = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); std::vector data_gen(nb * dim); for (int64_t i = 0; i < nb * dim; ++i) { @@ -216,7 +217,7 @@ test_run() { auto log_path = get_binlog_path(0); auto cm_w = ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::map> remote_files; std::map num_rows; diff --git a/internal/core/src/clustering/analyze_c.cpp b/internal/core/src/clustering/analyze_c.cpp index 61a519202b..fe163dad28 100644 --- a/internal/core/src/clustering/analyze_c.cpp +++ b/internal/core/src/clustering/analyze_c.cpp @@ -21,6 +21,7 @@ #include "types.h" #include "index/Utils.h" #include "index/Meta.h" +#include "storage/StorageV2FSCache.h" #include "storage/Util.h" #include "pb/clustering.pb.h" #include "monitor/scope_metric.h" @@ -78,9 +79,29 @@ Analyze(CAnalyze* res_analyze, get_storage_config(analyze_info->storage_config()); auto chunk_manager = milvus::storage::CreateChunkManager(storage_config); + auto fs = milvus::storage::StorageV2FSCache::Instance().Get({ + storage_config.address, + storage_config.bucket_name, + storage_config.access_key_id, + storage_config.access_key_value, + storage_config.root_path, + storage_config.storage_type, + storage_config.cloud_provider, + storage_config.iam_endpoint, + storage_config.log_level, + storage_config.region, + storage_config.useSSL, + storage_config.sslCACert, + storage_config.useIAM, + storage_config.useVirtualHost, + storage_config.requestTimeoutMs, + false, + storage_config.gcp_credential_json, + false, + }); milvus::storage::FileManagerContext fileManagerContext( - field_meta, index_meta, chunk_manager); + field_meta, index_meta, chunk_manager, fs); if (field_type != DataType::VECTOR_FLOAT) { throw SegcoreError( diff --git a/internal/core/src/exec/expression/JsonContainsByStatsTest.cpp b/internal/core/src/exec/expression/JsonContainsByStatsTest.cpp index 0f5d2fdbca..7b50393679 100644 --- a/internal/core/src/exec/expression/JsonContainsByStatsTest.cpp +++ b/internal/core/src/exec/expression/JsonContainsByStatsTest.cpp @@ -77,6 +77,7 @@ BuildAndLoadJsonKeyStats(const std::vector& json_strings, storage_config.storage_type = "local"; storage_config.root_path = root_path; auto chunk_manager = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); milvus_storage::ArrowFileSystemSingleton::GetInstance().Init( milvus_storage::ArrowFileSystemConfig{ @@ -94,7 +95,7 @@ BuildAndLoadJsonKeyStats(const std::vector& json_strings, chunk_manager->Write( log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager); + storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager, fs); Config build_config; build_config[INSERT_FILES_KEY] = std::vector{log_path}; diff --git a/internal/core/src/index/BitmapIndexArrayTest.cpp b/internal/core/src/index/BitmapIndexArrayTest.cpp index e03a0e4f00..2a8811d78a 100644 --- a/internal/core/src/index/BitmapIndexArrayTest.cpp +++ b/internal/core/src/index/BitmapIndexArrayTest.cpp @@ -17,6 +17,7 @@ #include "common/Tracer.h" #include "index/BitmapIndex.h" +#include "milvus-storage/filesystem/fs.h" #include "storage/Util.h" #include "storage/InsertData.h" #include "indexbuilder/IndexFactory.h" @@ -225,7 +226,8 @@ class ArrayBitmapIndexTest : public testing::Test { chunk_manager_->Write( log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_); + storage::FileManagerContext ctx( + field_meta, index_meta, chunk_manager_, fs_); std::vector index_files; Config config; @@ -297,6 +299,7 @@ class ArrayBitmapIndexTest : public testing::Test { storage_config.storage_type = "local"; storage_config.root_path = root_path; chunk_manager_ = storage::CreateChunkManager(storage_config); + fs_ = storage::InitArrowFileSystem(storage_config); Init(collection_id, partition_id, @@ -350,6 +353,7 @@ class ArrayBitmapIndexTest : public testing::Test { private: std::shared_ptr chunk_manager_; + milvus_storage::ArrowFileSystemPtr fs_; public: DataType type_; diff --git a/internal/core/src/index/BitmapIndexTest.cpp b/internal/core/src/index/BitmapIndexTest.cpp index be8d9068b5..a551a8e320 100644 --- a/internal/core/src/index/BitmapIndexTest.cpp +++ b/internal/core/src/index/BitmapIndexTest.cpp @@ -21,6 +21,7 @@ #include "common/Tracer.h" #include "common/Types.h" #include "index/BitmapIndex.h" +#include "milvus-storage/filesystem/fs.h" #include "storage/Util.h" #include "storage/InsertData.h" #include "indexbuilder/IndexFactory.h" @@ -156,7 +157,8 @@ class BitmapIndexTest : public testing::Test { chunk_manager_->Write( log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_); + storage::FileManagerContext ctx( + field_meta, index_meta, chunk_manager_, fs_); Config config; config["index_type"] = milvus::index::BITMAP_INDEX_TYPE; @@ -234,6 +236,7 @@ class BitmapIndexTest : public testing::Test { storage_config.storage_type = "local"; storage_config.root_path = root_path; chunk_manager_ = storage::CreateChunkManager(storage_config); + fs_ = storage::InitArrowFileSystem(storage_config); Init(collection_id, partition_id, @@ -518,6 +521,7 @@ class BitmapIndexTest : public testing::Test { bool nullable_; FixedVector valid_data_; std::shared_ptr chunk_manager_; + milvus_storage::ArrowFileSystemPtr fs_; int index_version_; int index_build_id_; bool has_default_value_{false}; diff --git a/internal/core/src/index/HybridScalarIndexTest.cpp b/internal/core/src/index/HybridScalarIndexTest.cpp index c629ffc539..940b7a4b86 100644 --- a/internal/core/src/index/HybridScalarIndexTest.cpp +++ b/internal/core/src/index/HybridScalarIndexTest.cpp @@ -154,7 +154,8 @@ class HybridIndexTestV1 : public testing::Test { chunk_manager_->Write( log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_); + storage::FileManagerContext ctx( + field_meta, index_meta, chunk_manager_, fs_); std::vector index_files; Config config; @@ -226,6 +227,7 @@ class HybridIndexTestV1 : public testing::Test { storage_config.storage_type = "local"; storage_config.root_path = root_path; chunk_manager_ = storage::CreateChunkManager(storage_config); + fs_ = storage::InitArrowFileSystem(storage_config); Init(collection_id, partition_id, @@ -507,6 +509,7 @@ class HybridIndexTestV1 : public testing::Test { size_t cardinality_; boost::container::vector data_; std::shared_ptr chunk_manager_; + milvus_storage::ArrowFileSystemPtr fs_; bool nullable_; FixedVector valid_data_; int index_build_id_; diff --git a/internal/core/src/index/InvertedIndexTest.cpp b/internal/core/src/index/InvertedIndexTest.cpp index f8c9f63938..0118aee66e 100644 --- a/internal/core/src/index/InvertedIndexTest.cpp +++ b/internal/core/src/index/InvertedIndexTest.cpp @@ -96,6 +96,7 @@ test_run() { std::string root_path = "/tmp/test-inverted-index/"; auto storage_config = gen_local_storage_config(root_path); auto cm = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); size_t nb = 10000; std::vector data_gen; @@ -160,7 +161,7 @@ test_run() { auto cm_w = test::ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::vector index_files; { @@ -498,6 +499,7 @@ test_string() { std::string root_path = "/tmp/test-inverted-index/"; auto storage_config = gen_local_storage_config(root_path); auto cm = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); size_t nb = 10000; boost::container::vector data; @@ -553,7 +555,7 @@ test_string() { auto cm_w = test::ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::vector index_files; { diff --git a/internal/core/src/index/JsonFlatIndexTest.cpp b/internal/core/src/index/JsonFlatIndexTest.cpp index deb0d30dc4..4c4840e337 100644 --- a/internal/core/src/index/JsonFlatIndexTest.cpp +++ b/internal/core/src/index/JsonFlatIndexTest.cpp @@ -19,6 +19,7 @@ #include "common/Tracer.h" #include "expr/ITypeExpr.h" #include "index/JsonFlatIndex.h" +#include "milvus-storage/filesystem/fs.h" #include "pb/plan.pb.h" #include "plan/PlanNode.h" #include "query/ExecPlanNodeVisitor.h" @@ -82,6 +83,7 @@ class JsonFlatIndexTest : public ::testing::Test { std::string root_path = "/tmp/test-json-flat-index/"; auto storage_config = gen_local_storage_config(root_path); cm_ = storage::CreateChunkManager(storage_config); + fs_ = storage::InitArrowFileSystem(storage_config); json_data_ = { R"({"profile": {"name": {"first": "Alice", "last": "Smith", "preferred_name": "Al"}, "team": {"name": "Engineering", "supervisor": {"name": "Bob"}}, "is_active": true, "employee_id": 1001, "skills": ["cpp", "rust", "python"], "scores": [95, 88, 92]}})", @@ -121,7 +123,7 @@ class JsonFlatIndexTest : public ::testing::Test { log_path_, serialized_bytes.data(), serialized_bytes.size()); ctx_ = std::make_unique( - field_meta_, index_meta_, cm_); + field_meta_, index_meta_, cm_, fs_); // Build index Config config; @@ -166,6 +168,7 @@ class JsonFlatIndexTest : public ::testing::Test { storage::FieldDataMeta field_meta_; storage::IndexMeta index_meta_; storage::ChunkManagerPtr cm_; + milvus_storage::ArrowFileSystemPtr fs_; std::unique_ptr cm_w_; std::unique_ptr ctx_; std::string log_path_; diff --git a/internal/core/src/index/NgramInvertedIndexTest.cpp b/internal/core/src/index/NgramInvertedIndexTest.cpp index b9e88057cb..815e454dfe 100644 --- a/internal/core/src/index/NgramInvertedIndexTest.cpp +++ b/internal/core/src/index/NgramInvertedIndexTest.cpp @@ -58,6 +58,7 @@ test_ngram_with_data(const boost::container::vector& data, std::string root_path = "/tmp/test-inverted-index/"; auto storage_config = gen_local_storage_config(root_path); auto cm = CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); size_t nb = data.size(); @@ -96,7 +97,7 @@ test_ngram_with_data(const boost::container::vector& data, auto cm_w = ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::vector index_files; auto index_size = 0; @@ -384,6 +385,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { std::string root_path = "/tmp/test-inverted-index/"; auto storage_config = gen_local_storage_config(root_path); auto cm = CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); size_t nb = data.size(); @@ -422,7 +424,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { auto cm_w = ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::vector index_files; // Build ngram index diff --git a/internal/core/src/segcore/FieldIndexing.cpp b/internal/core/src/segcore/FieldIndexing.cpp index 2c17414b84..b45b0179df 100644 --- a/internal/core/src/segcore/FieldIndexing.cpp +++ b/internal/core/src/segcore/FieldIndexing.cpp @@ -354,6 +354,8 @@ ScalarFieldIndexing::recreate_index(const FieldMeta& field_meta, auto chunk_manager = milvus::storage::LocalChunkManagerSingleton::GetInstance() .GetChunkManager(); + auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); // Create FieldDataMeta for RTree index storage::FieldDataMeta field_data_meta; @@ -380,7 +382,7 @@ ScalarFieldIndexing::recreate_index(const FieldMeta& field_meta, // Create FileManagerContext with all required components storage::FileManagerContext ctx( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs); index_ = std::make_unique>(ctx); built_ = false; diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp index d51fc57464..fa1f6d2d96 100644 --- a/internal/core/src/segcore/load_index_c.cpp +++ b/internal/core/src/segcore/load_index_c.cpp @@ -98,68 +98,6 @@ AppendIndexParam(CLoadIndexInfo c_load_index_info, } } -CStatus -appendVecIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { - SCOPE_CGO_CALL_METRIC(); - - try { - auto load_index_info = - (milvus::segcore::LoadIndexInfo*)c_load_index_info; - auto binary_set = (knowhere::BinarySet*)c_binary_set; - auto& index_params = load_index_info->index_params; - - milvus::index::CreateIndexInfo index_info; - index_info.field_type = load_index_info->field_type; - index_info.index_engine_version = load_index_info->index_engine_version; - - // get index type - AssertInfo(index_params.find("index_type") != index_params.end(), - "index type is empty"); - index_info.index_type = index_params.at("index_type"); - - // get metric type - AssertInfo(index_params.find("metric_type") != index_params.end(), - "metric type is empty"); - index_info.metric_type = index_params.at("metric_type"); - - // init file manager - milvus::storage::FieldDataMeta field_meta{ - load_index_info->collection_id, - load_index_info->partition_id, - load_index_info->segment_id, - load_index_info->field_id}; - milvus::storage::IndexMeta index_meta{load_index_info->segment_id, - load_index_info->field_id, - load_index_info->index_build_id, - load_index_info->index_version}; - auto remote_chunk_manager = - milvus::storage::RemoteChunkManagerSingleton::GetInstance() - .GetRemoteChunkManager(); - - auto config = milvus::index::ParseConfigFromIndexParams( - load_index_info->index_params); - config["index_files"] = load_index_info->index_files; - - milvus::storage::FileManagerContext fileManagerContext( - field_meta, index_meta, remote_chunk_manager); - fileManagerContext.set_for_loading_index(true); - - load_index_info->index = - milvus::index::IndexFactory::GetInstance().CreateIndex( - index_info, fileManagerContext); - load_index_info->index->Load(*binary_set, config); - auto status = CStatus(); - status.error_code = milvus::Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = milvus::UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - CStatus appendScalarIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { SCOPE_CGO_CALL_METRIC(); @@ -400,8 +338,11 @@ AppendIndexV2(CTraceContext c_trace, CLoadIndexInfo c_load_index_info) { auto remote_chunk_manager = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); + auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + AssertInfo(fs != nullptr, "arrow file system is nullptr"); milvus::storage::FileManagerContext fileManagerContext( - field_meta, index_meta, remote_chunk_manager); + field_meta, index_meta, remote_chunk_manager, fs); fileManagerContext.set_for_loading_index(true); // use cache layer to load vector/scalar index diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index c53809c261..13668961c5 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -477,6 +477,9 @@ LoadTextIndex(CSegmentInterface c_segment, auto remote_chunk_manager = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); + auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + AssertInfo(fs != nullptr, "arrow file system is null"); milvus::Config config; std::vector files; @@ -487,7 +490,7 @@ LoadTextIndex(CSegmentInterface c_segment, config[milvus::LOAD_PRIORITY] = info_proto->load_priority(); config[milvus::index::ENABLE_MMAP] = info_proto->enable_mmap(); milvus::storage::FileManagerContext ctx( - field_meta, index_meta, remote_chunk_manager); + field_meta, index_meta, remote_chunk_manager, fs); auto index = std::make_unique(ctx); index->Load(config); @@ -536,6 +539,9 @@ LoadJsonKeyIndex(CTraceContext c_trace, auto remote_chunk_manager = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); + auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); + AssertInfo(fs != nullptr, "arrow file system is null"); milvus::Config config; std::vector files; @@ -556,7 +562,7 @@ LoadJsonKeyIndex(CTraceContext c_trace, info_proto->fieldid(), info_proto->stats_size()}; milvus::storage::FileManagerContext file_ctx( - field_meta, index_meta, remote_chunk_manager); + field_meta, index_meta, remote_chunk_manager, fs); std::unique_ptr< milvus::cachinglayer::Translator> diff --git a/internal/core/src/segcore/storagev1translator/V1SealedIndexTranslator.cpp b/internal/core/src/segcore/storagev1translator/V1SealedIndexTranslator.cpp index f69dc8b463..8cbd06bbae 100644 --- a/internal/core/src/segcore/storagev1translator/V1SealedIndexTranslator.cpp +++ b/internal/core/src/segcore/storagev1translator/V1SealedIndexTranslator.cpp @@ -116,13 +116,15 @@ V1SealedIndexTranslator::LoadVecIndex() { auto remote_chunk_manager = milvus::storage::RemoteChunkManagerSingleton::GetInstance() .GetRemoteChunkManager(); + auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); auto config = milvus::index::ParseConfigFromIndexParams( index_load_info_.index_params); config["index_files"] = index_load_info_.index_files; milvus::storage::FileManagerContext fileManagerContext( - field_meta, index_meta, remote_chunk_manager); + field_meta, index_meta, remote_chunk_manager, fs); fileManagerContext.set_for_loading_index(true); auto index = milvus::index::IndexFactory::GetInstance().CreateIndex( diff --git a/internal/core/src/storage/DiskFileManagerTest.cpp b/internal/core/src/storage/DiskFileManagerTest.cpp index cb2e07b22a..0ad9d5022e 100644 --- a/internal/core/src/storage/DiskFileManagerTest.cpp +++ b/internal/core/src/storage/DiskFileManagerTest.cpp @@ -31,6 +31,7 @@ #include "common/Slice.h" #include "common/Common.h" #include "common/Types.h" +#include "milvus-storage/filesystem/fs.h" #include "storage/ChunkManager.h" #include "storage/DataCodec.h" #include "storage/InsertData.h" @@ -56,11 +57,14 @@ class DiskAnnFileManagerTest : public testing::Test { virtual void SetUp() { - cm_ = storage::CreateChunkManager(get_default_local_storage_config()); + auto storage_config = get_default_local_storage_config(); + cm_ = storage::CreateChunkManager(storage_config); + fs_ = storage::InitArrowFileSystem(storage_config); } protected: ChunkManagerPtr cm_; + milvus_storage::ArrowFileSystemPtr fs_; }; TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) { @@ -80,7 +84,7 @@ TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) { int64_t slice_size = milvus::FILE_SLICE_SIZE; auto diskAnnFileManager = std::make_shared( - storage::FileManagerContext(filed_data_meta, index_meta, cm_)); + storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs_)); auto ok = diskAnnFileManager->AddFile(indexFilePath); EXPECT_EQ(ok, true); @@ -161,7 +165,7 @@ TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) { IndexMeta index_meta = {3, 100, 1000, 1, "index"}; auto diskAnnFileManager = std::make_shared( - storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs)); + storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs_)); auto os = diskAnnFileManager->OpenOutputStream(index_file_path); size_t write_offset = 0; @@ -319,15 +323,16 @@ const FieldDataMeta kOptVecFieldDataMeta = {1, 2, 3, 100}; using OffsetT = uint32_t; auto -CreateFileManager(const ChunkManagerPtr& cm) +CreateFileManager(const ChunkManagerPtr& cm, + milvus_storage::ArrowFileSystemPtr fs) -> std::shared_ptr { // collection_id: 1, partition_id: 2, segment_id: 3 // field_id: 100, index_build_id: 1000, index_version: 1 IndexMeta index_meta = { 3, 100, 1000, 1, "opt_fields", "field_name", DataType::VECTOR_FLOAT, 1}; int64_t slice_size = milvus::FILE_SLICE_SIZE; - return std::make_shared( - storage::FileManagerContext(kOptVecFieldDataMeta, index_meta, cm)); + return std::make_shared(storage::FileManagerContext( + kOptVecFieldDataMeta, index_meta, cm, std::move(fs))); } template @@ -453,7 +458,7 @@ CheckOptFieldCorrectness( } // namespace TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOptFieldMoreThanOne) { - auto file_manager = CreateFileManager(cm_); + auto file_manager = CreateFileManager(cm_, fs_); const auto insert_file_path = PrepareInsertData(kOptFieldDataRange); OptFieldT opt_fields = @@ -468,7 +473,7 @@ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOptFieldMoreThanOne) { } TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskSpaceCorrect) { - auto file_manager = CreateFileManager(cm_); + auto file_manager = CreateFileManager(cm_, fs_); const auto insert_file_path = PrepareInsertData(kOptFieldDataRange); auto opt_fields = @@ -482,7 +487,7 @@ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskSpaceCorrect) { #define TEST_TYPE(NAME, TYPE, NATIVE_TYPE, RANGE) \ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskCorrect##NAME) { \ - auto file_manager = CreateFileManager(cm_); \ + auto file_manager = CreateFileManager(cm_, fs_); \ auto insert_file_path = PrepareInsertData(RANGE); \ auto opt_fields = \ PrepareOptionalField(file_manager, insert_file_path); \ @@ -505,7 +510,7 @@ TEST_TYPE(VARCHAR, DataType::VARCHAR, std::string, 100); #undef TEST_TYPE TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOnlyOneCategory) { - auto file_manager = CreateFileManager(cm_); + auto file_manager = CreateFileManager(cm_, fs_); { const auto insert_file_path = PrepareInsertData(1); @@ -527,7 +532,7 @@ TEST_F(DiskAnnFileManagerTest, FileCleanup) { LocalChunkManagerSingleton::GetInstance().GetChunkManager(); { - auto file_manager = CreateFileManager(cm_); + auto file_manager = CreateFileManager(cm_, fs_); auto random_file_suffix = std::to_string(rand()); local_text_index_file_path = diff --git a/internal/core/src/storage/FileManager.h b/internal/core/src/storage/FileManager.h index 0fc5ea107f..4beaa1172e 100644 --- a/internal/core/src/storage/FileManager.h +++ b/internal/core/src/storage/FileManager.h @@ -33,17 +33,17 @@ namespace milvus::storage { struct FileManagerContext { FileManagerContext() : chunkManagerPtr(nullptr) { } - FileManagerContext(const ChunkManagerPtr& chunkManagerPtr) + explicit FileManagerContext(const ChunkManagerPtr& chunkManagerPtr) : chunkManagerPtr(chunkManagerPtr) { } FileManagerContext(const FieldDataMeta& fieldDataMeta, const IndexMeta& indexMeta, const ChunkManagerPtr& chunkManagerPtr, - milvus_storage::ArrowFileSystemPtr fs = nullptr) + milvus_storage::ArrowFileSystemPtr fs) : fieldDataMeta(fieldDataMeta), indexMeta(indexMeta), chunkManagerPtr(chunkManagerPtr), - fs(fs) { + fs(std::move(fs)) { } bool diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index 2968691615..8c0d7326f8 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -18,6 +18,7 @@ #include "indexbuilder/IndexFactory.h" #include "indexbuilder/VecIndexCreator.h" #include "common/QueryResult.h" +#include "milvus-storage/filesystem/fs.h" #include "test_utils/indexbuilder_test_utils.h" #include "test_utils/storage_test_utils.h" @@ -32,6 +33,7 @@ class IndexWrapperTest : public ::testing::TestWithParam { void SetUp() override { storage_config_ = get_default_local_storage_config(); + fs_ = storage::InitArrowFileSystem(storage_config_); auto param = GetParam(); index_type = param.first; @@ -98,6 +100,7 @@ class IndexWrapperTest : public ::testing::TestWithParam { int64_t query_offset = 1; int64_t NB = 10; StorageConfig storage_config_; + milvus_storage::ArrowFileSystemPtr fs_; }; INSTANTIATE_TEST_SUITE_P( @@ -126,7 +129,7 @@ TEST_P(IndexWrapperTest, BuildAndQuery) { auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_); storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs_); config[milvus::index::INDEX_ENGINE_VERSION] = std::to_string(knowhere::Version::GetCurrentVersion().VersionNumber()); auto index = milvus::indexbuilder::IndexFactory::GetInstance().CreateIndex( diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index fb6be4356d..2400ea4276 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -311,6 +311,7 @@ class IndexTest : public ::testing::TestWithParam { void SetUp() override { storage_config_ = get_default_local_storage_config(); + fs_ = milvus::storage::InitArrowFileSystem(storage_config_); auto param = GetParam(); index_type = param.first; @@ -393,6 +394,7 @@ class IndexTest : public ::testing::TestWithParam { int64_t query_offset = 100; int64_t NB = 3000; // will be updated to 27000 for mmap+hnsw StorageConfig storage_config_; + milvus_storage::ArrowFileSystemPtr fs_; }; INSTANTIATE_TEST_SUITE_P( @@ -485,7 +487,7 @@ TEST_P(IndexTest, BuildAndQuery) { milvus::storage::IndexMeta index_meta{3, 100, 1000, 1}; auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs_); index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -554,7 +556,7 @@ TEST_P(IndexTest, Mmap) { milvus::storage::IndexMeta index_meta{3, 100, 1000, 1}; auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs_); index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -617,7 +619,7 @@ TEST_P(IndexTest, GetVector) { milvus::storage::IndexMeta index_meta{3, 100, 1000, 1}; auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs_); index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -720,7 +722,7 @@ TEST_P(IndexTest, GetVector_EmptySparseVector) { milvus::storage::IndexMeta index_meta{3, 100, 1000, 1}; auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs_); index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -788,8 +790,9 @@ TEST(Indexing, SearchDiskAnnWithInvalidParam) { milvus::storage::IndexMeta index_meta{ segment_id, field_id, build_id, index_version}; auto chunk_manager = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs); auto index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -874,8 +877,9 @@ TEST(Indexing, SearchDiskAnnWithFloat16) { milvus::storage::IndexMeta index_meta{ segment_id, field_id, build_id, index_version}; auto chunk_manager = storage::CreateChunkManager(storage_config); + auto fs = storage::InitArrowFileSystem(storage_config); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs); auto index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); @@ -960,8 +964,9 @@ TEST(Indexing, SearchDiskAnnWithBFloat16) { milvus::storage::IndexMeta index_meta{ segment_id, field_id, build_id, index_version}; auto chunk_manager = storage::CreateChunkManager(storage_config); + auto fs = milvus::storage::InitArrowFileSystem(storage_config); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs); auto index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, file_manager_context); diff --git a/internal/core/unittest/test_json_stats/test_json_key_stats.cpp b/internal/core/unittest/test_json_stats/test_json_key_stats.cpp index 006f703575..eaae69fedc 100644 --- a/internal/core/unittest/test_json_stats/test_json_key_stats.cpp +++ b/internal/core/unittest/test_json_stats/test_json_key_stats.cpp @@ -18,6 +18,7 @@ #include "common/Tracer.h" #include "index/BitmapIndex.h" +#include "milvus-storage/filesystem/fs.h" #include "storage/Util.h" #include "storage/InsertData.h" #include "indexbuilder/IndexFactory.h" @@ -134,7 +135,8 @@ class JsonKeyStatsTest : public ::testing::TestWithParam { chunk_manager_->Write( log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_); + storage::FileManagerContext ctx( + field_meta, index_meta, chunk_manager_, fs_); std::vector index_files; Config config; @@ -183,6 +185,8 @@ class JsonKeyStatsTest : public ::testing::TestWithParam { conf.storage_type = "local"; conf.root_path = "/tmp/test-jsonkey-stats/arrow-fs"; milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf); + fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); Init(collection_id, partition_id, @@ -206,6 +210,7 @@ class JsonKeyStatsTest : public ::testing::TestWithParam { std::vector data_; std::vector json_col; std::shared_ptr chunk_manager_; + milvus_storage::ArrowFileSystemPtr fs_; }; INSTANTIATE_TEST_SUITE_P(JsonKeyStatsTestSuite, @@ -348,6 +353,8 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test { conf.storage_type = "local"; conf.root_path = "/tmp/test-jsonkey-stats-upload-load/arrow-fs"; milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf); + fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance() + .GetArrowFileSystem(); } void @@ -402,7 +409,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test { log_path, serialized_bytes.data(), serialized_bytes.size()); storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); Config config; config[INSERT_FILES_KEY] = std::vector{log_path}; @@ -421,7 +428,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test { void Load() { storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); Config config; config["index_files"] = index_files_; config[milvus::index::ENABLE_MMAP] = true; @@ -475,6 +482,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test { std::shared_ptr build_index_; std::shared_ptr load_index_; std::vector index_files_; + milvus_storage::ArrowFileSystemPtr fs_; }; TEST_F(JsonKeyStatsUploadLoadTest, TestSimpleJson) { diff --git a/internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp b/internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp index 0d200d82a0..1d076de694 100644 --- a/internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp +++ b/internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp @@ -89,7 +89,8 @@ TEST(TraverseJsonForBuildStatsTest, storage_config.storage_type = "local"; storage_config.root_path = "/tmp/test-traverse-json-build-stats"; auto cm = milvus::storage::CreateChunkManager(storage_config); - milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm); + auto fs = milvus::storage::InitArrowFileSystem(storage_config); + milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); JsonKeyStats stats(ctx, true); int index = 0; diff --git a/internal/core/unittest/test_rtree_index.cpp b/internal/core/unittest/test_rtree_index.cpp index 1c2afd15d4..a3ded7f265 100644 --- a/internal/core/unittest/test_rtree_index.cpp +++ b/internal/core/unittest/test_rtree_index.cpp @@ -111,6 +111,8 @@ class RTreeIndexTest : public ::testing::Test { storage_config_.root_path = temp_path_.get().string(); chunk_manager_ = milvus::storage::CreateChunkManager(storage_config_); + fs_ = milvus::storage::InitArrowFileSystem(storage_config_); + // prepare field & index meta – minimal info for DiskFileManagerImpl field_meta_ = milvus::storage::FieldDataMeta{1, 1, 1, 100}; // set geometry data type in field schema for index schema checks @@ -170,12 +172,13 @@ class RTreeIndexTest : public ::testing::Test { milvus::storage::FieldDataMeta field_meta_; milvus::storage::IndexMeta index_meta_; milvus::test::TmpPath temp_path_; + milvus_storage::ArrowFileSystemPtr fs_; }; TEST_F(RTreeIndexTest, Build_Upload_Load) { // ---------- Build via BuildWithRawDataForUT ---------- milvus::storage::FileManagerContext ctx_build( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree_build(ctx_build); std::vector wkbs = {CreatePointWKB(1.0, 1.0), @@ -191,7 +194,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) { // ---------- Load back ---------- milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -207,7 +210,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) { TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) { // Build & upload first milvus::storage::FileManagerContext ctx_build( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree_build(ctx_build); std::vector wkbs2 = {CreatePointWKB(10.0, 10.0), @@ -228,7 +231,7 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) { // Load using filename only list milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -243,7 +246,7 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) { TEST_F(RTreeIndexTest, Build_EmptyInput_ShouldThrow) { milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); std::vector empty; @@ -253,7 +256,7 @@ TEST_F(RTreeIndexTest, Build_EmptyInput_ShouldThrow) { TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) { milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); std::string bad = CreatePointWKB(0.0, 0.0); @@ -267,7 +270,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) { auto stats = rtree.Upload({}); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -282,7 +285,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) { TEST_F(RTreeIndexTest, Build_VariousGeometries) { milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); std::vector wkbs = { @@ -299,7 +302,7 @@ TEST_F(RTreeIndexTest, Build_VariousGeometries) { ASSERT_FALSE(stats->GetIndexFiles().empty()); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -317,7 +320,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) { auto remote_file = (temp_path_.get() / "geom.parquet").string(); WriteGeometryInsertFile(chunk_manager_, field_meta_, remote_file, wkbs); milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); nlohmann::json build_cfg; @@ -328,7 +331,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) { // Cache remote index files locally milvus::storage::DiskFileManagerImpl diskfm( - {field_meta_, index_meta_, chunk_manager_}); + {field_meta_, index_meta_, chunk_manager_, fs_}); auto index_files = stats->GetIndexFiles(); auto load_priority = milvus::index::GetValueFromConfig( @@ -376,7 +379,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) { TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) { // Build and upload milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); std::vector wkbs = {CreatePointWKB(6.0, 6.0), CreatePointWKB(7.0, 7.0)}; @@ -389,7 +392,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) { mixed[0] = boost::filesystem::path(mixed[0]).filename().string(); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -402,7 +405,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) { TEST_F(RTreeIndexTest, Load_NonexistentRemote_ShouldThrow) { milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -422,7 +425,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) { WriteGeometryInsertFile(chunk_manager_, field_meta_, remote_file, wkbs); milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); nlohmann::json build_cfg; @@ -434,7 +437,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) { auto stats = rtree.Upload({}); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); nlohmann::json cfg; @@ -461,7 +464,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) { // Build from insert_files (not using BuildWithRawDataForUT) milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); nlohmann::json build_cfg; @@ -477,7 +480,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) { // Load index back and verify milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -515,7 +518,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) { // build (default to bulk load) milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); nlohmann::json build_cfg; @@ -531,7 +534,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) { ASSERT_GT(stats->GetIndexFiles().size(), 0); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); @@ -548,7 +551,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) { TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) { // Build a small index in-memory (via UT API) milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); // Prepare simple geometries: two points and a square polygon @@ -564,7 +567,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) { // Upload and then load into a new index instance for querying auto stats = rtree.Upload({}); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); nlohmann::json cfg; @@ -622,7 +625,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) { TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) { milvus::storage::FileManagerContext ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); milvus::index::RTreeIndex rtree(ctx); // Two overlapping squares and one disjoint square @@ -640,7 +643,7 @@ TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) { // Upload and load a new instance for querying auto stats = rtree.Upload({}); milvus::storage::FileManagerContext ctx_load( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); ctx_load.set_for_loading_index(true); milvus::index::RTreeIndex rtree_load(ctx_load); nlohmann::json cfg; @@ -762,7 +765,7 @@ TEST_F(RTreeIndexTest, GIS_Index_Exact_Filtering) { // build index files by invoking RTreeIndex::Build milvus::storage::FileManagerContext fm_ctx( - field_meta_, index_meta_, chunk_manager_); + field_meta_, index_meta_, chunk_manager_, fs_); auto rtree_index = std::make_unique>(fm_ctx); nlohmann::json build_cfg; diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index b338e9d947..74d55cfb60 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -2396,6 +2396,7 @@ TEST(Sealed, SearchVectorArray) { std::string root_path = "/tmp/test-vector-array/"; auto storage_config = gen_local_storage_config(root_path); auto cm = CreateChunkManager(storage_config); + auto fs = milvus::storage::InitArrowFileSystem(storage_config); auto vec_array_col = dataset.get_col(array_vec); std::vector vector_arrays; for (auto& v : vec_array_col) { @@ -2438,7 +2439,7 @@ TEST(Sealed, SearchVectorArray) { auto cm_w = ChunkManagerWrapper(cm); cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size()); - storage::FileManagerContext ctx(field_meta, index_meta, cm); + storage::FileManagerContext ctx(field_meta, index_meta, cm, fs); std::vector index_files; // create index @@ -2452,7 +2453,7 @@ TEST(Sealed, SearchVectorArray) { auto emb_list_hnsw_index = milvus::index::IndexFactory::GetInstance().CreateIndex( create_index_info, - storage::FileManagerContext(field_meta, index_meta, cm)); + storage::FileManagerContext(field_meta, index_meta, cm, fs)); // build index Config config; diff --git a/internal/core/unittest/test_storage_v2_index_raw_data.cpp b/internal/core/unittest/test_storage_v2_index_raw_data.cpp index 61d20286a8..9f182c9732 100644 --- a/internal/core/unittest/test_storage_v2_index_raw_data.cpp +++ b/internal/core/unittest/test_storage_v2_index_raw_data.cpp @@ -109,7 +109,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) { false); auto index_meta = gen_index_meta( segment_id, int64_field.get(), index_build_id, index_version); - storage::FileManagerContext ctx(field_meta, index_meta, cm_); + storage::FileManagerContext ctx(field_meta, index_meta, cm_, fs_); auto index = indexbuilder::IndexFactory::GetInstance().CreateIndex( milvus::DataType::INT64, config, ctx); @@ -148,7 +148,8 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) { collection_id, partition_id, segment_id, float_field.get()}; auto file_manager = std::make_shared( - storage::FileManagerContext(field_data_meta, index_meta, cm_)); + storage::FileManagerContext( + field_data_meta, index_meta, cm_, fs_)); auto res = file_manager->CacheRawDataToDisk(config); ASSERT_EQ(res, "/tmp/milvus/local_data/raw_datas/3/105/raw_data"); } @@ -175,7 +176,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) { FieldDataMeta field_data_meta = { collection_id, partition_id, segment_id, int32_field.get()}; auto ctx = - storage::FileManagerContext(field_data_meta, index_meta, cm_); + storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_); auto int32_index = milvus::index::CreateScalarIndexSort(ctx); int32_index->Build(config); @@ -204,7 +205,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) { FieldDataMeta field_data_meta = { collection_id, partition_id, segment_id, varchar_field.get()}; auto ctx = - storage::FileManagerContext(field_data_meta, index_meta, cm_); + storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_); auto string_index = std::make_unique(ctx); @@ -237,7 +238,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) { FieldDataMeta field_data_meta = { collection_id, partition_id, segment_id, vec_field.get()}; auto ctx = - storage::FileManagerContext(field_data_meta, index_meta, cm_); + storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_); try { auto vec_index = diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index 5ce035ad78..8bf2ddd5c7 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -1587,8 +1587,9 @@ GenVecIndexing(int64_t N, storage_config.storage_type = "local"; storage_config.root_path = TestRemotePath; auto chunk_manager = milvus::storage::CreateChunkManager(storage_config); + auto fs = milvus::storage::InitArrowFileSystem(storage_config); milvus::storage::FileManagerContext file_manager_context( - field_data_meta, index_meta, chunk_manager); + field_data_meta, index_meta, chunk_manager, fs); auto indexing = std::make_unique>( DataType::NONE, index_type, diff --git a/internal/querynodev2/segments/segment_loader_test.go b/internal/querynodev2/segments/segment_loader_test.go index d87c9945da..b96e6c432e 100644 --- a/internal/querynodev2/segments/segment_loader_test.go +++ b/internal/querynodev2/segments/segment_loader_test.go @@ -86,6 +86,8 @@ func (suite *SegmentLoaderSuite) SetupTest() { initcore.InitLocalChunkManager(suite.rootPath) initcore.InitMmapManager(paramtable.Get(), 1) initcore.InitTieredStorage(paramtable.Get()) + initcore.InitLocalArrowFileSystem(suite.rootPath) + initcore.InitRemoteArrowFileSystem(paramtable.Get()) // Data suite.schema = mock_segcore.GenTestCollectionSchema("test", schemapb.DataType_Int64, false)