admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: [2.6] Pass fs via FileManagerContext when loading index (#44734)

Browse Source 
Cherry-pick from master
pr: #44733
Related to #44615

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2025-10-11 09:57:57 +08:00 committed by GitHub
parent a85b06d965
commit 07bca45376
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 159 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/clustering/KmeansClusteringTest.cpp
View File

@ -189,6 +189,7 @@ test_run() {
std::string root_path = "/tmp/test-kmeans-clustering/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
std::vector<T> data_gen(nb * dim);
for (int64_t i = 0; i < nb * dim; ++i) {
@ -216,7 +217,7 @@ test_run() {
auto log_path = get_binlog_path(0);
auto cm_w = ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::map<int64_t, std::vector<std::string>> remote_files;
std::map<int64_t, int64_t> num_rows;

23
internal/core/src/clustering/analyze_c.cpp
View File

@ -21,6 +21,7 @@
#include "types.h"
#include "index/Utils.h"
#include "index/Meta.h"
#include "storage/StorageV2FSCache.h"
#include "storage/Util.h"
#include "pb/clustering.pb.h"
#include "monitor/scope_metric.h"
@ -78,9 +79,29 @@ Analyze(CAnalyze* res_analyze,
get_storage_config(analyze_info->storage_config());
auto chunk_manager =
milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::StorageV2FSCache::Instance().Get({
storage_config.address,
storage_config.bucket_name,
storage_config.access_key_id,
storage_config.access_key_value,
storage_config.root_path,
storage_config.storage_type,
storage_config.cloud_provider,
storage_config.iam_endpoint,
storage_config.log_level,
storage_config.region,
storage_config.useSSL,
storage_config.sslCACert,
storage_config.useIAM,
storage_config.useVirtualHost,
storage_config.requestTimeoutMs,
false,
storage_config.gcp_credential_json,
false,
});
milvus::storage::FileManagerContext fileManagerContext(
field_meta, index_meta, chunk_manager);
field_meta, index_meta, chunk_manager, fs);
if (field_type != DataType::VECTOR_FLOAT) {
throw SegcoreError(

3
internal/core/src/exec/expression/JsonContainsByStatsTest.cpp
View File

@ -77,6 +77,7 @@ BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
storage_config.storage_type = "local";
storage_config.root_path = root_path;
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(
milvus_storage::ArrowFileSystemConfig{
@ -94,7 +95,7 @@ BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
chunk_manager->Write(
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager);
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager, fs);
Config build_config;
build_config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};

6
internal/core/src/index/BitmapIndexArrayTest.cpp
View File

@ -17,6 +17,7 @@
#include "common/Tracer.h"
#include "index/BitmapIndex.h"
#include "milvus-storage/filesystem/fs.h"
#include "storage/Util.h"
#include "storage/InsertData.h"
#include "indexbuilder/IndexFactory.h"
@ -225,7 +226,8 @@ class ArrayBitmapIndexTest : public testing::Test {
chunk_manager_->Write(
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_);
storage::FileManagerContext ctx(
field_meta, index_meta, chunk_manager_, fs_);
std::vector<std::string> index_files;
Config config;
@ -297,6 +299,7 @@ class ArrayBitmapIndexTest : public testing::Test {
storage_config.storage_type = "local";
storage_config.root_path = root_path;
chunk_manager_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
Init(collection_id,
partition_id,
@ -350,6 +353,7 @@ class ArrayBitmapIndexTest : public testing::Test {
private:
std::shared_ptr<storage::ChunkManager> chunk_manager_;
milvus_storage::ArrowFileSystemPtr fs_;
public:
DataType type_;

6
internal/core/src/index/BitmapIndexTest.cpp
View File

@ -21,6 +21,7 @@
#include "common/Tracer.h"
#include "common/Types.h"
#include "index/BitmapIndex.h"
#include "milvus-storage/filesystem/fs.h"
#include "storage/Util.h"
#include "storage/InsertData.h"
#include "indexbuilder/IndexFactory.h"
@ -156,7 +157,8 @@ class BitmapIndexTest : public testing::Test {
chunk_manager_->Write(
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_);
storage::FileManagerContext ctx(
field_meta, index_meta, chunk_manager_, fs_);
Config config;
config["index_type"] = milvus::index::BITMAP_INDEX_TYPE;
@ -234,6 +236,7 @@ class BitmapIndexTest : public testing::Test {
storage_config.storage_type = "local";
storage_config.root_path = root_path;
chunk_manager_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
Init(collection_id,
partition_id,
@ -518,6 +521,7 @@ class BitmapIndexTest : public testing::Test {
bool nullable_;
FixedVector<bool> valid_data_;
std::shared_ptr<storage::ChunkManager> chunk_manager_;
milvus_storage::ArrowFileSystemPtr fs_;
int index_version_;
int index_build_id_;
bool has_default_value_{false};

5
internal/core/src/index/HybridScalarIndexTest.cpp
View File

@ -154,7 +154,8 @@ class HybridIndexTestV1 : public testing::Test {
chunk_manager_->Write(
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_);
storage::FileManagerContext ctx(
field_meta, index_meta, chunk_manager_, fs_);
std::vector<std::string> index_files;
Config config;
@ -226,6 +227,7 @@ class HybridIndexTestV1 : public testing::Test {
storage_config.storage_type = "local";
storage_config.root_path = root_path;
chunk_manager_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
Init(collection_id,
partition_id,
@ -507,6 +509,7 @@ class HybridIndexTestV1 : public testing::Test {
size_t cardinality_;
boost::container::vector<T> data_;
std::shared_ptr<storage::ChunkManager> chunk_manager_;
milvus_storage::ArrowFileSystemPtr fs_;
bool nullable_;
FixedVector<bool> valid_data_;
int index_build_id_;

6
internal/core/src/index/InvertedIndexTest.cpp
View File

@ -96,6 +96,7 @@ test_run() {
std::string root_path = "/tmp/test-inverted-index/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
size_t nb = 10000;
std::vector<T> data_gen;
@ -160,7 +161,7 @@ test_run() {
auto cm_w = test::ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::vector<std::string> index_files;
{
@ -498,6 +499,7 @@ test_string() {
std::string root_path = "/tmp/test-inverted-index/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
size_t nb = 10000;
boost::container::vector<T> data;
@ -553,7 +555,7 @@ test_string() {
auto cm_w = test::ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::vector<std::string> index_files;
{

5
internal/core/src/index/JsonFlatIndexTest.cpp
View File

@ -19,6 +19,7 @@
#include "common/Tracer.h"
#include "expr/ITypeExpr.h"
#include "index/JsonFlatIndex.h"
#include "milvus-storage/filesystem/fs.h"
#include "pb/plan.pb.h"
#include "plan/PlanNode.h"
#include "query/ExecPlanNodeVisitor.h"
@ -82,6 +83,7 @@ class JsonFlatIndexTest : public ::testing::Test {
std::string root_path = "/tmp/test-json-flat-index/";
auto storage_config = gen_local_storage_config(root_path);
cm_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
json_data_ = {
R"({"profile": {"name": {"first": "Alice", "last": "Smith", "preferred_name": "Al"}, "team": {"name": "Engineering", "supervisor": {"name": "Bob"}}, "is_active": true, "employee_id": 1001, "skills": ["cpp", "rust", "python"], "scores": [95, 88, 92]}})",
@ -121,7 +123,7 @@ class JsonFlatIndexTest : public ::testing::Test {
log_path_, serialized_bytes.data(), serialized_bytes.size());
ctx_ = std::make_unique<storage::FileManagerContext>(
field_meta_, index_meta_, cm_);
field_meta_, index_meta_, cm_, fs_);
// Build index
Config config;
@ -166,6 +168,7 @@ class JsonFlatIndexTest : public ::testing::Test {
storage::FieldDataMeta field_meta_;
storage::IndexMeta index_meta_;
storage::ChunkManagerPtr cm_;
milvus_storage::ArrowFileSystemPtr fs_;
std::unique_ptr<test::ChunkManagerWrapper> cm_w_;
std::unique_ptr<storage::FileManagerContext> ctx_;
std::string log_path_;

6
internal/core/src/index/NgramInvertedIndexTest.cpp
View File

@ -58,6 +58,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
std::string root_path = "/tmp/test-inverted-index/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
size_t nb = data.size();
@ -96,7 +97,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
auto cm_w = ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::vector<std::string> index_files;
auto index_size = 0;
@ -384,6 +385,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
std::string root_path = "/tmp/test-inverted-index/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
size_t nb = data.size();
@ -422,7 +424,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
auto cm_w = ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::vector<std::string> index_files;
// Build ngram index

4
internal/core/src/segcore/FieldIndexing.cpp
View File

@ -354,6 +354,8 @@ ScalarFieldIndexing<T>::recreate_index(const FieldMeta& field_meta,
auto chunk_manager =
milvus::storage::LocalChunkManagerSingleton::GetInstance()
.GetChunkManager();
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
// Create FieldDataMeta for RTree index
storage::FieldDataMeta field_data_meta;
@ -380,7 +382,7 @@ ScalarFieldIndexing<T>::recreate_index(const FieldMeta& field_meta,
// Create FileManagerContext with all required components
storage::FileManagerContext ctx(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs);
index_ = std::make_unique<index::RTreeIndex<std::string>>(ctx);
built_ = false;

67
internal/core/src/segcore/load_index_c.cpp
View File

@ -98,68 +98,6 @@ AppendIndexParam(CLoadIndexInfo c_load_index_info,
}
}
CStatus
appendVecIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
SCOPE_CGO_CALL_METRIC();
try {
auto load_index_info =
(milvus::segcore::LoadIndexInfo*)c_load_index_info;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
auto& index_params = load_index_info->index_params;
milvus::index::CreateIndexInfo index_info;
index_info.field_type = load_index_info->field_type;
index_info.index_engine_version = load_index_info->index_engine_version;
// get index type
AssertInfo(index_params.find("index_type") != index_params.end(),
"index type is empty");
index_info.index_type = index_params.at("index_type");
// get metric type
AssertInfo(index_params.find("metric_type") != index_params.end(),
"metric type is empty");
index_info.metric_type = index_params.at("metric_type");
// init file manager
milvus::storage::FieldDataMeta field_meta{
load_index_info->collection_id,
load_index_info->partition_id,
load_index_info->segment_id,
load_index_info->field_id};
milvus::storage::IndexMeta index_meta{load_index_info->segment_id,
load_index_info->field_id,
load_index_info->index_build_id,
load_index_info->index_version};
auto remote_chunk_manager =
milvus::storage::RemoteChunkManagerSingleton::GetInstance()
.GetRemoteChunkManager();
auto config = milvus::index::ParseConfigFromIndexParams(
load_index_info->index_params);
config["index_files"] = load_index_info->index_files;
milvus::storage::FileManagerContext fileManagerContext(
field_meta, index_meta, remote_chunk_manager);
fileManagerContext.set_for_loading_index(true);
load_index_info->index =
milvus::index::IndexFactory::GetInstance().CreateIndex(
index_info, fileManagerContext);
load_index_info->index->Load(*binary_set, config);
auto status = CStatus();
status.error_code = milvus::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = milvus::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
appendScalarIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
SCOPE_CGO_CALL_METRIC();
@ -400,8 +338,11 @@ AppendIndexV2(CTraceContext c_trace, CLoadIndexInfo c_load_index_info) {
auto remote_chunk_manager =
milvus::storage::RemoteChunkManagerSingleton::GetInstance()
.GetRemoteChunkManager();
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
AssertInfo(fs != nullptr, "arrow file system is nullptr");
milvus::storage::FileManagerContext fileManagerContext(
field_meta, index_meta, remote_chunk_manager);
field_meta, index_meta, remote_chunk_manager, fs);
fileManagerContext.set_for_loading_index(true);
// use cache layer to load vector/scalar index

10
internal/core/src/segcore/segment_c.cpp
View File

@ -477,6 +477,9 @@ LoadTextIndex(CSegmentInterface c_segment,
auto remote_chunk_manager =
milvus::storage::RemoteChunkManagerSingleton::GetInstance()
.GetRemoteChunkManager();
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
AssertInfo(fs != nullptr, "arrow file system is null");
milvus::Config config;
std::vector<std::string> files;
@ -487,7 +490,7 @@ LoadTextIndex(CSegmentInterface c_segment,
config[milvus::LOAD_PRIORITY] = info_proto->load_priority();
config[milvus::index::ENABLE_MMAP] = info_proto->enable_mmap();
milvus::storage::FileManagerContext ctx(
field_meta, index_meta, remote_chunk_manager);
field_meta, index_meta, remote_chunk_manager, fs);
auto index = std::make_unique<milvus::index::TextMatchIndex>(ctx);
index->Load(config);
@ -536,6 +539,9 @@ LoadJsonKeyIndex(CTraceContext c_trace,
auto remote_chunk_manager =
milvus::storage::RemoteChunkManagerSingleton::GetInstance()
.GetRemoteChunkManager();
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
AssertInfo(fs != nullptr, "arrow file system is null");
milvus::Config config;
std::vector<std::string> files;
@ -556,7 +562,7 @@ LoadJsonKeyIndex(CTraceContext c_trace,
info_proto->fieldid(),
info_proto->stats_size()};
milvus::storage::FileManagerContext file_ctx(
field_meta, index_meta, remote_chunk_manager);
field_meta, index_meta, remote_chunk_manager, fs);
std::unique_ptr<
milvus::cachinglayer::Translator<milvus::index::JsonKeyStats>>

4
internal/core/src/segcore/storagev1translator/V1SealedIndexTranslator.cpp
View File

@ -116,13 +116,15 @@ V1SealedIndexTranslator::LoadVecIndex() {
auto remote_chunk_manager =
milvus::storage::RemoteChunkManagerSingleton::GetInstance()
.GetRemoteChunkManager();
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
auto config = milvus::index::ParseConfigFromIndexParams(
index_load_info_.index_params);
config["index_files"] = index_load_info_.index_files;
milvus::storage::FileManagerContext fileManagerContext(
field_meta, index_meta, remote_chunk_manager);
field_meta, index_meta, remote_chunk_manager, fs);
fileManagerContext.set_for_loading_index(true);
auto index = milvus::index::IndexFactory::GetInstance().CreateIndex(

27
internal/core/src/storage/DiskFileManagerTest.cpp
View File

@ -31,6 +31,7 @@
#include "common/Slice.h"
#include "common/Common.h"
#include "common/Types.h"
#include "milvus-storage/filesystem/fs.h"
#include "storage/ChunkManager.h"
#include "storage/DataCodec.h"
#include "storage/InsertData.h"
@ -56,11 +57,14 @@ class DiskAnnFileManagerTest : public testing::Test {
virtual void
SetUp() {
cm_ = storage::CreateChunkManager(get_default_local_storage_config());
auto storage_config = get_default_local_storage_config();
cm_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
}
protected:
ChunkManagerPtr cm_;
milvus_storage::ArrowFileSystemPtr fs_;
};
TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) {
@ -80,7 +84,7 @@ TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) {
int64_t slice_size = milvus::FILE_SLICE_SIZE;
auto diskAnnFileManager = std::make_shared<DiskFileManagerImpl>(
storage::FileManagerContext(filed_data_meta, index_meta, cm_));
storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs_));
auto ok = diskAnnFileManager->AddFile(indexFilePath);
EXPECT_EQ(ok, true);
@ -161,7 +165,7 @@ TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) {
IndexMeta index_meta = {3, 100, 1000, 1, "index"};
auto diskAnnFileManager = std::make_shared<DiskFileManagerImpl>(
storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs));
storage::FileManagerContext(filed_data_meta, index_meta, cm_, fs_));
auto os = diskAnnFileManager->OpenOutputStream(index_file_path);
size_t write_offset = 0;
@ -319,15 +323,16 @@ const FieldDataMeta kOptVecFieldDataMeta = {1, 2, 3, 100};
using OffsetT = uint32_t;
auto
CreateFileManager(const ChunkManagerPtr& cm)
CreateFileManager(const ChunkManagerPtr& cm,
milvus_storage::ArrowFileSystemPtr fs)
-> std::shared_ptr<DiskFileManagerImpl> {
// collection_id: 1, partition_id: 2, segment_id: 3
// field_id: 100, index_build_id: 1000, index_version: 1
IndexMeta index_meta = {
3, 100, 1000, 1, "opt_fields", "field_name", DataType::VECTOR_FLOAT, 1};
int64_t slice_size = milvus::FILE_SLICE_SIZE;
return std::make_shared<DiskFileManagerImpl>(
storage::FileManagerContext(kOptVecFieldDataMeta, index_meta, cm));
return std::make_shared<DiskFileManagerImpl>(storage::FileManagerContext(
kOptVecFieldDataMeta, index_meta, cm, std::move(fs)));
}
template <typename T>
@ -453,7 +458,7 @@ CheckOptFieldCorrectness(
} // namespace
TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOptFieldMoreThanOne) {
auto file_manager = CreateFileManager(cm_);
auto file_manager = CreateFileManager(cm_, fs_);
const auto insert_file_path =
PrepareInsertData<DataType::INT64, int64_t>(kOptFieldDataRange);
OptFieldT opt_fields =
@ -468,7 +473,7 @@ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOptFieldMoreThanOne) {
}
TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskSpaceCorrect) {
auto file_manager = CreateFileManager(cm_);
auto file_manager = CreateFileManager(cm_, fs_);
const auto insert_file_path =
PrepareInsertData<DataType::INT64, int64_t>(kOptFieldDataRange);
auto opt_fields =
@ -482,7 +487,7 @@ TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskSpaceCorrect) {
#define TEST_TYPE(NAME, TYPE, NATIVE_TYPE, RANGE) \
TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskCorrect##NAME) { \
auto file_manager = CreateFileManager(cm_); \
auto file_manager = CreateFileManager(cm_, fs_); \
auto insert_file_path = PrepareInsertData<TYPE, NATIVE_TYPE>(RANGE); \
auto opt_fields = \
PrepareOptionalField<TYPE>(file_manager, insert_file_path); \
@ -505,7 +510,7 @@ TEST_TYPE(VARCHAR, DataType::VARCHAR, std::string, 100);
#undef TEST_TYPE
TEST_F(DiskAnnFileManagerTest, CacheOptFieldToDiskOnlyOneCategory) {
auto file_manager = CreateFileManager(cm_);
auto file_manager = CreateFileManager(cm_, fs_);
{
const auto insert_file_path =
PrepareInsertData<DataType::INT64, int64_t>(1);
@ -527,7 +532,7 @@ TEST_F(DiskAnnFileManagerTest, FileCleanup) {
LocalChunkManagerSingleton::GetInstance().GetChunkManager();
{
auto file_manager = CreateFileManager(cm_);
auto file_manager = CreateFileManager(cm_, fs_);
auto random_file_suffix = std::to_string(rand());
local_text_index_file_path =

6
internal/core/src/storage/FileManager.h
View File

@ -33,17 +33,17 @@ namespace milvus::storage {
struct FileManagerContext {
FileManagerContext() : chunkManagerPtr(nullptr) {
}
FileManagerContext(const ChunkManagerPtr& chunkManagerPtr)
explicit FileManagerContext(const ChunkManagerPtr& chunkManagerPtr)
: chunkManagerPtr(chunkManagerPtr) {
}
FileManagerContext(const FieldDataMeta& fieldDataMeta,
const IndexMeta& indexMeta,
const ChunkManagerPtr& chunkManagerPtr,
milvus_storage::ArrowFileSystemPtr fs = nullptr)
milvus_storage::ArrowFileSystemPtr fs)
: fieldDataMeta(fieldDataMeta),
indexMeta(indexMeta),
chunkManagerPtr(chunkManagerPtr),
fs(fs) {
fs(std::move(fs)) {
}
bool

5
internal/core/unittest/test_index_wrapper.cpp
View File

@ -18,6 +18,7 @@
#include "indexbuilder/IndexFactory.h"
#include "indexbuilder/VecIndexCreator.h"
#include "common/QueryResult.h"
#include "milvus-storage/filesystem/fs.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
@ -32,6 +33,7 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
void
SetUp() override {
storage_config_ = get_default_local_storage_config();
fs_ = storage::InitArrowFileSystem(storage_config_);
auto param = GetParam();
index_type = param.first;
@ -98,6 +100,7 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
int64_t query_offset = 1;
int64_t NB = 10;
StorageConfig storage_config_;
milvus_storage::ArrowFileSystemPtr fs_;
};
INSTANTIATE_TEST_SUITE_P(
@ -126,7 +129,7 @@ TEST_P(IndexWrapperTest, BuildAndQuery) {
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_);
storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs_);
config[milvus::index::INDEX_ENGINE_VERSION] =
std::to_string(knowhere::Version::GetCurrentVersion().VersionNumber());
auto index = milvus::indexbuilder::IndexFactory::GetInstance().CreateIndex(

19
internal/core/unittest/test_indexing.cpp
View File

@ -311,6 +311,7 @@ class IndexTest : public ::testing::TestWithParam<Param> {
void
SetUp() override {
storage_config_ = get_default_local_storage_config();
fs_ = milvus::storage::InitArrowFileSystem(storage_config_);
auto param = GetParam();
index_type = param.first;
@ -393,6 +394,7 @@ class IndexTest : public ::testing::TestWithParam<Param> {
int64_t query_offset = 100;
int64_t NB = 3000; // will be updated to 27000 for mmap+hnsw
StorageConfig storage_config_;
milvus_storage::ArrowFileSystemPtr fs_;
};
INSTANTIATE_TEST_SUITE_P(
@ -485,7 +487,7 @@ TEST_P(IndexTest, BuildAndQuery) {
milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs_);
index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -554,7 +556,7 @@ TEST_P(IndexTest, Mmap) {
milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs_);
index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -617,7 +619,7 @@ TEST_P(IndexTest, GetVector) {
milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs_);
index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -720,7 +722,7 @@ TEST_P(IndexTest, GetVector_EmptySparseVector) {
milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config_);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs_);
index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -788,8 +790,9 @@ TEST(Indexing, SearchDiskAnnWithInvalidParam) {
milvus::storage::IndexMeta index_meta{
segment_id, field_id, build_id, index_version};
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs);
auto index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -874,8 +877,9 @@ TEST(Indexing, SearchDiskAnnWithFloat16) {
milvus::storage::IndexMeta index_meta{
segment_id, field_id, build_id, index_version};
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs);
auto index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);
@ -960,8 +964,9 @@ TEST(Indexing, SearchDiskAnnWithBFloat16) {
milvus::storage::IndexMeta index_meta{
segment_id, field_id, build_id, index_version};
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs);
auto index = milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info, file_manager_context);

14
internal/core/unittest/test_json_stats/test_json_key_stats.cpp
View File

@ -18,6 +18,7 @@
#include "common/Tracer.h"
#include "index/BitmapIndex.h"
#include "milvus-storage/filesystem/fs.h"
#include "storage/Util.h"
#include "storage/InsertData.h"
#include "indexbuilder/IndexFactory.h"
@ -134,7 +135,8 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
chunk_manager_->Write(
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager_);
storage::FileManagerContext ctx(
field_meta, index_meta, chunk_manager_, fs_);
std::vector<std::string> index_files;
Config config;
@ -183,6 +185,8 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
conf.storage_type = "local";
conf.root_path = "/tmp/test-jsonkey-stats/arrow-fs";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
Init(collection_id,
partition_id,
@ -206,6 +210,7 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
std::vector<milvus::Json> data_;
std::vector<std::string> json_col;
std::shared_ptr<storage::ChunkManager> chunk_manager_;
milvus_storage::ArrowFileSystemPtr fs_;
};
INSTANTIATE_TEST_SUITE_P(JsonKeyStatsTestSuite,
@ -348,6 +353,8 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
conf.storage_type = "local";
conf.root_path = "/tmp/test-jsonkey-stats-upload-load/arrow-fs";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
}
void
@ -402,7 +409,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
Config config;
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
@ -421,7 +428,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
void
Load() {
storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
Config config;
config["index_files"] = index_files_;
config[milvus::index::ENABLE_MMAP] = true;
@ -475,6 +482,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
std::shared_ptr<JsonKeyStats> build_index_;
std::shared_ptr<JsonKeyStats> load_index_;
std::vector<std::string> index_files_;
milvus_storage::ArrowFileSystemPtr fs_;
};
TEST_F(JsonKeyStatsUploadLoadTest, TestSimpleJson) {

3
internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp
View File

@ -89,7 +89,8 @@ TEST(TraverseJsonForBuildStatsTest,
storage_config.storage_type = "local";
storage_config.root_path = "/tmp/test-traverse-json-build-stats";
auto cm = milvus::storage::CreateChunkManager(storage_config);
milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
JsonKeyStats stats(ctx, true);
int index = 0;

53
internal/core/unittest/test_rtree_index.cpp
View File

@ -111,6 +111,8 @@ class RTreeIndexTest : public ::testing::Test {
storage_config_.root_path = temp_path_.get().string();
chunk_manager_ = milvus::storage::CreateChunkManager(storage_config_);
fs_ = milvus::storage::InitArrowFileSystem(storage_config_);
// prepare field & index meta – minimal info for DiskFileManagerImpl
field_meta_ = milvus::storage::FieldDataMeta{1, 1, 1, 100};
// set geometry data type in field schema for index schema checks
@ -170,12 +172,13 @@ class RTreeIndexTest : public ::testing::Test {
milvus::storage::FieldDataMeta field_meta_;
milvus::storage::IndexMeta index_meta_;
milvus::test::TmpPath temp_path_;
milvus_storage::ArrowFileSystemPtr fs_;
};
TEST_F(RTreeIndexTest, Build_Upload_Load) {
// ---------- Build via BuildWithRawDataForUT ----------
milvus::storage::FileManagerContext ctx_build(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree_build(ctx_build);
std::vector<std::string> wkbs = {CreatePointWKB(1.0, 1.0),
@ -191,7 +194,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) {
// ---------- Load back ----------
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -207,7 +210,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) {
TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) {
// Build & upload first
milvus::storage::FileManagerContext ctx_build(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree_build(ctx_build);
std::vector<std::string> wkbs2 = {CreatePointWKB(10.0, 10.0),
@ -228,7 +231,7 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) {
// Load using filename only list
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -243,7 +246,7 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) {
TEST_F(RTreeIndexTest, Build_EmptyInput_ShouldThrow) {
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
std::vector<std::string> empty;
@ -253,7 +256,7 @@ TEST_F(RTreeIndexTest, Build_EmptyInput_ShouldThrow) {
TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) {
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
std::string bad = CreatePointWKB(0.0, 0.0);
@ -267,7 +270,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) {
auto stats = rtree.Upload({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -282,7 +285,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) {
TEST_F(RTreeIndexTest, Build_VariousGeometries) {
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
std::vector<std::string> wkbs = {
@ -299,7 +302,7 @@ TEST_F(RTreeIndexTest, Build_VariousGeometries) {
ASSERT_FALSE(stats->GetIndexFiles().empty());
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -317,7 +320,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) {
auto remote_file = (temp_path_.get() / "geom.parquet").string();
WriteGeometryInsertFile(chunk_manager_, field_meta_, remote_file, wkbs);
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
nlohmann::json build_cfg;
@ -328,7 +331,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) {
// Cache remote index files locally
milvus::storage::DiskFileManagerImpl diskfm(
{field_meta_, index_meta_, chunk_manager_});
{field_meta_, index_meta_, chunk_manager_, fs_});
auto index_files = stats->GetIndexFiles();
auto load_priority =
milvus::index::GetValueFromConfig<milvus::proto::common::LoadPriority>(
@ -376,7 +379,7 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) {
TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
// Build and upload
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
std::vector<std::string> wkbs = {CreatePointWKB(6.0, 6.0),
CreatePointWKB(7.0, 7.0)};
@ -389,7 +392,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
mixed[0] = boost::filesystem::path(mixed[0]).filename().string();
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -402,7 +405,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
TEST_F(RTreeIndexTest, Load_NonexistentRemote_ShouldThrow) {
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -422,7 +425,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) {
WriteGeometryInsertFile(chunk_manager_, field_meta_, remote_file, wkbs);
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
nlohmann::json build_cfg;
@ -434,7 +437,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) {
auto stats = rtree.Upload({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
nlohmann::json cfg;
@ -461,7 +464,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) {
// Build from insert_files (not using BuildWithRawDataForUT)
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
nlohmann::json build_cfg;
@ -477,7 +480,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) {
// Load index back and verify
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -515,7 +518,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) {
// build (default to bulk load)
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
nlohmann::json build_cfg;
@ -531,7 +534,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) {
ASSERT_GT(stats->GetIndexFiles().size(), 0);
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
@ -548,7 +551,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) {
TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) {
// Build a small index in-memory (via UT API)
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
// Prepare simple geometries: two points and a square polygon
@ -564,7 +567,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) {
// Upload and then load into a new index instance for querying
auto stats = rtree.Upload({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
nlohmann::json cfg;
@ -622,7 +625,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) {
TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) {
milvus::storage::FileManagerContext ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
milvus::index::RTreeIndex<std::string> rtree(ctx);
// Two overlapping squares and one disjoint square
@ -640,7 +643,7 @@ TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) {
// Upload and load a new instance for querying
auto stats = rtree.Upload({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
nlohmann::json cfg;
@ -762,7 +765,7 @@ TEST_F(RTreeIndexTest, GIS_Index_Exact_Filtering) {
// build index files by invoking RTreeIndex::Build
milvus::storage::FileManagerContext fm_ctx(
field_meta_, index_meta_, chunk_manager_);
field_meta_, index_meta_, chunk_manager_, fs_);
auto rtree_index =
std::make_unique<milvus::index::RTreeIndex<std::string>>(fm_ctx);
nlohmann::json build_cfg;

5
internal/core/unittest/test_sealed.cpp
View File

@ -2396,6 +2396,7 @@ TEST(Sealed, SearchVectorArray) {
std::string root_path = "/tmp/test-vector-array/";
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
auto vec_array_col = dataset.get_col<VectorFieldProto>(array_vec);
std::vector<milvus::VectorArray> vector_arrays;
for (auto& v : vec_array_col) {
@ -2438,7 +2439,7 @@ TEST(Sealed, SearchVectorArray) {
auto cm_w = ChunkManagerWrapper(cm);
cm_w.Write(log_path, serialized_bytes.data(), serialized_bytes.size());
storage::FileManagerContext ctx(field_meta, index_meta, cm);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
std::vector<std::string> index_files;
// create index
@ -2452,7 +2453,7 @@ TEST(Sealed, SearchVectorArray) {
auto emb_list_hnsw_index =
milvus::index::IndexFactory::GetInstance().CreateIndex(
create_index_info,
storage::FileManagerContext(field_meta, index_meta, cm));
storage::FileManagerContext(field_meta, index_meta, cm, fs));
// build index
Config config;

11
internal/core/unittest/test_storage_v2_index_raw_data.cpp
View File

@ -109,7 +109,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
false);
auto index_meta = gen_index_meta(
segment_id, int64_field.get(), index_build_id, index_version);
storage::FileManagerContext ctx(field_meta, index_meta, cm_);
storage::FileManagerContext ctx(field_meta, index_meta, cm_, fs_);
auto index = indexbuilder::IndexFactory::GetInstance().CreateIndex(
milvus::DataType::INT64, config, ctx);
@ -148,7 +148,8 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
collection_id, partition_id, segment_id, float_field.get()};
auto file_manager =
std::make_shared<milvus::storage::DiskFileManagerImpl>(
storage::FileManagerContext(field_data_meta, index_meta, cm_));
storage::FileManagerContext(
field_data_meta, index_meta, cm_, fs_));
auto res = file_manager->CacheRawDataToDisk<float>(config);
ASSERT_EQ(res, "/tmp/milvus/local_data/raw_datas/3/105/raw_data");
}
@ -175,7 +176,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
FieldDataMeta field_data_meta = {
collection_id, partition_id, segment_id, int32_field.get()};
auto ctx =
storage::FileManagerContext(field_data_meta, index_meta, cm_);
storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_);
auto int32_index = milvus::index::CreateScalarIndexSort<int32_t>(ctx);
int32_index->Build(config);
@ -204,7 +205,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
FieldDataMeta field_data_meta = {
collection_id, partition_id, segment_id, varchar_field.get()};
auto ctx =
storage::FileManagerContext(field_data_meta, index_meta, cm_);
storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_);
auto string_index =
std::make_unique<milvus::index::StringIndexMarisa>(ctx);
@ -237,7 +238,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
FieldDataMeta field_data_meta = {
collection_id, partition_id, segment_id, vec_field.get()};
auto ctx =
storage::FileManagerContext(field_data_meta, index_meta, cm_);
storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_);
try {
auto vec_index =

3
internal/core/unittest/test_utils/DataGen.h
View File

@ -1587,8 +1587,9 @@ GenVecIndexing(int64_t N,
storage_config.storage_type = "local";
storage_config.root_path = TestRemotePath;
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext file_manager_context(
field_data_meta, index_meta, chunk_manager);
field_data_meta, index_meta, chunk_manager, fs);
auto indexing = std::make_unique<index::VectorMemIndex<float>>(
DataType::NONE,
index_type,

2
internal/querynodev2/segments/segment_loader_test.go
View File

@ -86,6 +86,8 @@ func (suite *SegmentLoaderSuite) SetupTest() {
initcore.InitLocalChunkManager(suite.rootPath)
initcore.InitMmapManager(paramtable.Get(), 1)
initcore.InitTieredStorage(paramtable.Get())
initcore.InitLocalArrowFileSystem(suite.rootPath)
initcore.InitRemoteArrowFileSystem(paramtable.Get())
// Data
suite.schema = mock_segcore.GenTestCollectionSchema("test", schemapb.DataType_Int64, false)