admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 09:08:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: support mmap for jsonstats shared key index (#44914)

Browse Source 
#42533

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2025-11-27 16:01:08 +08:00 committed by GitHub
parent b6532d3e44
commit 1b58844319
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/src/index/json_stats/JsonKeyStats.cpp
View File

@ -1009,7 +1009,9 @@ JsonKeyStats::Load(milvus::tracer::TraceContext ctx, const Config& config) {
LoadShreddingData(shredding_data_files);
// load shared key index
bson_inverted_index_->LoadIndex(shared_key_index_files, load_priority_);
bson_inverted_index_->LoadIndex(shared_key_index_files,
load_priority_,
config.contains(MMAP_FILE_PATH));
}
IndexStatsPtr

18
internal/core/src/index/json_stats/bson_inverted.cpp
View File

@ -104,7 +104,8 @@ BsonInvertedIndex::BuildIndex() {
void
BsonInvertedIndex::LoadIndex(const std::vector<std::string>& index_files,
milvus::proto::common::LoadPriority priority) {
milvus::proto::common::LoadPriority priority,
bool load_in_mmap) {
if (is_load_) {
// convert shared_key_index/... to remote_prefix/shared_key_index/...
std::vector<std::string> remote_files;
@ -122,10 +123,17 @@ BsonInvertedIndex::LoadIndex(const std::vector<std::string>& index_files,
"index dir not exist: {}",
path_);
wrapper_ = std::make_shared<TantivyIndexWrapper>(
path_.c_str(), false, milvus::index::SetBitsetUnused);
LOG_INFO("load json shared key index done for field id:{} with dir:{}",
field_id_,
path_);
path_.c_str(), load_in_mmap, milvus::index::SetBitsetUnused);
if (!load_in_mmap) {
disk_file_manager_->RemoveJsonStatsSharedIndexFiles();
}
LOG_INFO(
"load json shared key index done for field id:{} with "
"dir:{},load_in_mmap:{}",
field_id_,
path_,
load_in_mmap);
}
}

3
internal/core/src/index/json_stats/bson_inverted.h
View File

@ -57,7 +57,8 @@ class BsonInvertedIndex {
void
LoadIndex(const std::vector<std::string>& index_files,
milvus::proto::common::LoadPriority priority);
milvus::proto::common::LoadPriority priority,
bool load_in_mmap);
IndexStatsPtr
UploadIndex();

7
internal/core/src/storage/DiskFileManagerImpl.cpp
View File

@ -738,6 +738,13 @@ DiskFileManagerImpl::RemoveTextLogFiles() {
local_chunk_manager->RemoveDir(GetLocalTextIndexPrefix());
}
void
DiskFileManagerImpl::RemoveJsonStatsSharedIndexFiles() {
auto local_chunk_manager =
LocalChunkManagerSingleton::GetInstance().GetChunkManager();
local_chunk_manager->RemoveDir(GetLocalJsonStatsSharedIndexPrefix());
}
void
DiskFileManagerImpl::RemoveJsonStatsFiles() {
auto local_chunk_manager =

3
internal/core/src/storage/DiskFileManagerImpl.h
View File

@ -173,6 +173,9 @@ class DiskFileManagerImpl : public FileManagerImpl {
void
RemoveTextLogFiles();
void
RemoveJsonStatsSharedIndexFiles();
void
RemoveJsonStatsFiles();