admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Dirty BloomFilter is left in the cache (#5252)

Browse Source 
Multi threads accessing will leave dirty BloomFilter in the cache.
Now, reading threads will not overwrite the cache. And it should be solved.
Issue 4897 involves BloomFilter and Blacklist.
This commit try to fix BloomFilter firstly.

Resolves: #4897

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>
This commit is contained in:
cqy123456 2021-05-18 11:02:10 +08:00 committed by GitHub
parent 0497fe46ae
commit dce238e939
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 37 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/cache/Cache.h vendored
View File

@ -66,6 +66,9 @@ class Cache {
void
insert(const std::string& key, const ItemObj& item);
void
insert_if_not_exist(const std::string& key, const ItemObj& item);
void
erase(const std::string& key);

10
core/src/cache/Cache.inl vendored
View File

@ -64,6 +64,16 @@ Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
insert_internal(key, item);
}
template <typename ItemObj>
void
Cache<ItemObj>::insert_if_not_exist(const std::string& key, const ItemObj& item) {
std::lock_guard<std::mutex> lock(mutex_);
if (lru_.exists(key) == true) {
return ;
}
insert_internal(key, item);
}
template <typename ItemObj>
void
Cache<ItemObj>::erase(const std::string& key) {

3
core/src/cache/CacheMgr.h vendored
View File

@ -36,6 +36,9 @@ class CacheMgr {
virtual void
InsertItem(const std::string& key, const ItemObj& data);
virtual void
InsertItemIfNotExist(const std::string& key, const ItemObj& data);
virtual void
EraseItem(const std::string& key);

11
core/src/cache/CacheMgr.inl vendored
View File

@ -62,6 +62,17 @@ CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::InsertItemIfNotExist(const std::string& key, const ItemObj& data) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->insert_if_not_exist(key, data);
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::EraseItem(const std::string& key) {

2
core/src/db/DBImpl.cpp
View File

@ -1502,7 +1502,7 @@ DBImpl::GetVectorsByIdHelper(const IDNumbers& id_array, std::vector<engine::Vect
engine::utils::GetParentPath(file.location_, segment_dir);
segment::SegmentReader segment_reader(segment_dir);
segment::IdBloomFilterPtr id_bloom_filter_ptr;
auto status = segment_reader.LoadBloomFilter(id_bloom_filter_ptr);
auto status = segment_reader.LoadBloomFilter(id_bloom_filter_ptr, false);
if (!status.ok()) {
return status;
}

2
core/src/db/insert/MemTable.cpp
View File

@ -232,7 +232,7 @@ MemTable::ApplyDeletes() {
utils::GetParentPath(file.location_, segment_dir);
segment::SegmentReader segment_reader(segment_dir);
segment_reader.LoadBloomFilter(id_bloom_filter_ptr);
segment_reader.LoadBloomFilter(id_bloom_filter_ptr, true);
for (auto& id : doc_ids_to_delete_) {
if (id_bloom_filter_ptr->Check(id)) {

2
core/src/scheduler/JobMgr.cpp
View File

@ -98,7 +98,7 @@ JobMgr::worker_function() {
engine::utils::GetParentPath(location, segment_dir);
segment::SegmentReader segment_reader(segment_dir);
segment::IdBloomFilterPtr id_bloom_filter_ptr;
segment_reader.LoadBloomFilter(id_bloom_filter_ptr);
segment_reader.LoadBloomFilter(id_bloom_filter_ptr, false);
// Check if the id is present.
bool pass = true;

8
core/src/segment/SegmentReader.cpp
View File

@ -104,7 +104,7 @@ SegmentReader::LoadVectorIndex(const std::string& location, segment::VectorIndex
}
Status
SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr, bool cache_force) {
codec::DefaultCodec default_codec;
try {
// load id_bloom_filter from cache
@ -117,7 +117,11 @@ SegmentReader::LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr) {
default_codec.GetIdBloomFilterFormat()->read(fs_ptr_, id_bloom_filter_ptr);
// add id_bloom_filter into cache
cache::CpuCacheMgr::GetInstance()->InsertItem(cache_key, id_bloom_filter_ptr);
if (cache_force) {
cache::CpuCacheMgr::GetInstance()->InsertItem(cache_key, id_bloom_filter_ptr);
} else {
cache::CpuCacheMgr::GetInstance()->InsertItemIfNotExist(cache_key, id_bloom_filter_ptr);
}
}
} catch (std::exception& e) {
std::string err_msg = "Failed to load bloom filter: " + std::string(e.what());

2
core/src/segment/SegmentReader.h
View File

@ -49,7 +49,7 @@ class SegmentReader {
LoadVectorIndex(const std::string& location, segment::VectorIndexPtr& vector_index_ptr);
Status
LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr);
LoadBloomFilter(segment::IdBloomFilterPtr& id_bloom_filter_ptr, bool cache_force);
Status
LoadDeletedDocs(segment::DeletedDocsPtr& deleted_docs_ptr);