admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: support mmap for jsonstats shared key index (#44914)

Browse Source 
#42533

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2025-11-27 16:01:08 +08:00 committed by GitHub
parent b6532d3e44
commit 1b58844319
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/src/index/json_stats/JsonKeyStats.cpp
View File

@ -1009,7 +1009,9 @@ JsonKeyStats::Load(milvus::tracer::TraceContext ctx, const Config& config) {
LoadShreddingData(shredding_data_files); LoadShreddingData(shredding_data_files);
// load shared key index // load shared key index
bson_inverted_index_->LoadIndex(shared_key_index_files, load_priority_); bson_inverted_index_->LoadIndex(shared_key_index_files,
load_priority_,
config.contains(MMAP_FILE_PATH));
} }
IndexStatsPtr IndexStatsPtr

16
internal/core/src/index/json_stats/bson_inverted.cpp
View File

@ -104,7 +104,8 @@ BsonInvertedIndex::BuildIndex() {
void void
BsonInvertedIndex::LoadIndex(const std::vector<std::string>& index_files, BsonInvertedIndex::LoadIndex(const std::vector<std::string>& index_files,
milvus::proto::common::LoadPriority priority) { milvus::proto::common::LoadPriority priority,
bool load_in_mmap) {
if (is_load_) { if (is_load_) {
// convert shared_key_index/... to remote_prefix/shared_key_index/... // convert shared_key_index/... to remote_prefix/shared_key_index/...
std::vector<std::string> remote_files; std::vector<std::string> remote_files;
@ -122,10 +123,17 @@ BsonInvertedIndex::LoadIndex(const std::vector<std::string>& index_files,
"index dir not exist: {}", "index dir not exist: {}",
path_); path_);
wrapper_ = std::make_shared<TantivyIndexWrapper>( wrapper_ = std::make_shared<TantivyIndexWrapper>(
path_.c_str(), false, milvus::index::SetBitsetUnused); path_.c_str(), load_in_mmap, milvus::index::SetBitsetUnused);
LOG_INFO("load json shared key index done for field id:{} with dir:{}",
if (!load_in_mmap) {
disk_file_manager_->RemoveJsonStatsSharedIndexFiles();
}
LOG_INFO(
"load json shared key index done for field id:{} with "
"dir:{},load_in_mmap:{}",
field_id_, field_id_,
path_); path_,
load_in_mmap);
} }
} }

3
internal/core/src/index/json_stats/bson_inverted.h
View File

@ -57,7 +57,8 @@ class BsonInvertedIndex {
void void
LoadIndex(const std::vector<std::string>& index_files, LoadIndex(const std::vector<std::string>& index_files,
milvus::proto::common::LoadPriority priority); milvus::proto::common::LoadPriority priority,
bool load_in_mmap);
IndexStatsPtr IndexStatsPtr
UploadIndex(); UploadIndex();

7
internal/core/src/storage/DiskFileManagerImpl.cpp
View File

@ -738,6 +738,13 @@ DiskFileManagerImpl::RemoveTextLogFiles() {
local_chunk_manager->RemoveDir(GetLocalTextIndexPrefix()); local_chunk_manager->RemoveDir(GetLocalTextIndexPrefix());
} }
void
DiskFileManagerImpl::RemoveJsonStatsSharedIndexFiles() {
auto local_chunk_manager =
LocalChunkManagerSingleton::GetInstance().GetChunkManager();
local_chunk_manager->RemoveDir(GetLocalJsonStatsSharedIndexPrefix());
}
void void
DiskFileManagerImpl::RemoveJsonStatsFiles() { DiskFileManagerImpl::RemoveJsonStatsFiles() {
auto local_chunk_manager = auto local_chunk_manager =

3
internal/core/src/storage/DiskFileManagerImpl.h
View File

@ -173,6 +173,9 @@ class DiskFileManagerImpl : public FileManagerImpl {
void void
RemoveTextLogFiles(); RemoveTextLogFiles();
void
RemoveJsonStatsSharedIndexFiles();
void void
RemoveJsonStatsFiles(); RemoveJsonStatsFiles();