From ef59efc76a8a60e6d939398c82fac7adfa09b649 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Fri, 19 Apr 2019 10:00:06 +0800 Subject: [PATCH 1/4] feat(db): remove mutex for meta operation Former-commit-id: 349c7856679f76eedd9c29f0208bd27beae93b33 --- cpp/src/db/DBMetaImpl.cpp | 12 +----------- cpp/src/db/DBMetaImpl.h | 4 ---- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 7a0bef88a3..126a64c8af 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -76,6 +76,7 @@ Status DBMetaImpl::initialize() { ConnectorPtr = std::make_unique(StoragePrototype(_options.path+"/meta.sqlite")); ConnectorPtr->sync_schema(); + ConnectorPtr->open_forever(); // thread safe option cleanup(); @@ -93,8 +94,6 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) { group_info.id = -1; { - std::unique_lock lk(mutex_); - try { auto id = ConnectorPtr->insert(group_info); group_info.id = id; @@ -114,7 +113,6 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) { } Status DBMetaImpl::get_group(GroupSchema& group_info) { - std::unique_lock lk(mutex_); return get_group_no_lock(group_info); } @@ -139,7 +137,6 @@ Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) { } Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { - std::unique_lock lk(mutex_); auto groups = ConnectorPtr->select(columns(&GroupSchema::id), where(c(&GroupSchema::group_id) == group_id)); assert(groups.size() <= 1); @@ -171,8 +168,6 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { GetGroupFilePath(group_file); { - std::unique_lock lk(mutex_); - try { auto id = ConnectorPtr->insert(group_file); group_file.id = id; @@ -194,7 +189,6 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { files.clear(); - std::unique_lock lk(mutex_); auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, &GroupFileSchema::group_id, &GroupFileSchema::file_id, @@ -235,7 +229,6 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, DatePartionedGroupFilesSchema& files) { files.clear(); - std::unique_lock lk(mutex_); auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, &GroupFileSchema::group_id, &GroupFileSchema::file_id, @@ -294,7 +287,6 @@ Status DBMetaImpl::get_group_files(const std::string& group_id_, } Status DBMetaImpl::update_group_file(const GroupFileSchema& group_file) { - std::unique_lock lk(mutex_); auto commited = ConnectorPtr->transaction([&] () mutable { ConnectorPtr->update(group_file); return true; @@ -306,7 +298,6 @@ Status DBMetaImpl::update_group_file(const GroupFileSchema& group_file) { } Status DBMetaImpl::update_files(const GroupFilesSchema& files) { - std::unique_lock lk(mutex_); auto commited = ConnectorPtr->transaction([&] () mutable { for (auto& file : files) { ConnectorPtr->update(file); @@ -320,7 +311,6 @@ Status DBMetaImpl::update_files(const GroupFilesSchema& files) { } Status DBMetaImpl::cleanup() { - std::unique_lock lk(mutex_); auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, &GroupFileSchema::group_id, &GroupFileSchema::file_id, diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 5fa10e986e..86d1cd56ba 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -1,8 +1,6 @@ #ifndef VECENGINE_DB_META_IMPL_H_ #define VECENGINE_DB_META_IMPL_H_ -#include -#include #include "Meta.h" #include "Options.h" @@ -55,8 +53,6 @@ private: Status initialize(); const DBMetaOptions _options; - std::mutex mutex_; - }; // DBMetaImpl } // namespace meta From 7c6a033a6ac5f34a482e6c4014791371ad13aa4d Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Fri, 19 Apr 2019 10:16:52 +0800 Subject: [PATCH 2/4] fix(db): set new file rows to 0 Former-commit-id: ca814fc5f6c81679e25ec4cd195b430a89b52383 --- cpp/src/db/DBImpl.cpp | 2 ++ cpp/src/db/DBMetaImpl.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index b92518d696..7dc61679c3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -133,6 +133,7 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, } else { group_file.file_type = meta::GroupFileSchema::RAW; } + group_file.rows = index_size; updated.push_back(group_file); status = _pMeta->update_files(updated); @@ -188,6 +189,7 @@ Status DBImpl::build_index(const meta::GroupFileSchema& file) { /* std::cout << "raw size=" << from_index->ntotal << " index size=" << index->ntotal << std::endl; */ write_index(index, group_file.location.c_str()); group_file.file_type = meta::GroupFileSchema::INDEX; + group_file.rows = file.dimension * index->ntotal; auto to_remove = file; to_remove.file_type = meta::GroupFileSchema::TO_DELETE; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 126a64c8af..3b0f520673 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -165,6 +165,7 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { group_file.file_type = GroupFileSchema::NEW; group_file.file_id = ss.str(); group_file.dimension = group_info.dimension; + group_file.rows = 0; GetGroupFilePath(group_file); { From 2a886ca1918fb0f938b63be038c069f1534419c6 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Fri, 19 Apr 2019 10:17:17 +0800 Subject: [PATCH 3/4] fix(db): no default sqlite dumped Former-commit-id: 763198a34f09cfecc6d768b0b6eb28a29e1bf288 --- cpp/src/db/db_connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/db_connection.cpp b/cpp/src/db/db_connection.cpp index 2870a9a447..74779daf18 100644 --- a/cpp/src/db/db_connection.cpp +++ b/cpp/src/db/db_connection.cpp @@ -24,7 +24,7 @@ SqliteDBPtr connect() { return temp; } -SqliteDBPtr Connection::connect_ = connect(); +/* SqliteDBPtr Connection::connect_ = connect(); */ } } From efeecf54e0fc1ca3b9b2f83dc1dda86bf265a043 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Fri, 19 Apr 2019 15:57:12 +0800 Subject: [PATCH 4/4] feat(db): cleanup pre db destruction Former-commit-id: 3eb7940392378a61e031d0b840d7b5e797e10f6e --- cpp/src/db/DBImpl.cpp | 24 ++++++++++++++++++------ cpp/src/db/DBImpl.h | 2 ++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 7dc61679c3..5fb02e0b83 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -107,7 +107,9 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, group_file.group_id = group_id; group_file.date = date; Status status = _pMeta->add_group_file(group_file); + if (!status.ok()) { + std::cout << status.ToString() << std::endl; return status; } @@ -127,7 +129,6 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, auto index_size = group_file.dimension * index->ntotal; faiss::write_index(index.get(), group_file.location.c_str()); - std::cout << "Merged size=" << index_size << std::endl; if (index_size >= _options.index_trigger_size) { group_file.file_type = meta::GroupFileSchema::TO_INDEX; } else { @@ -201,6 +202,7 @@ Status DBImpl::build_index(const meta::GroupFileSchema& file) { } void DBImpl::background_build_index() { + std::lock_guard lock(build_index_mutex_); assert(bg_build_index_started_); meta::GroupFilesSchema to_index_files; _pMeta->files_to_index(to_index_files); @@ -214,6 +216,7 @@ void DBImpl::background_build_index() { } bg_build_index_started_ = false; + bg_build_index_finish_signal_.notify_all(); } Status DBImpl::try_build_index() { @@ -230,7 +233,6 @@ void DBImpl::background_compaction() { Status status; for (auto group_id : group_ids) { - /* std::cout << __func__ << " group_id=" << group_id << std::endl; */ status = background_merge_files(group_id); if (!status.ok()) { _bg_error = status; @@ -240,11 +242,21 @@ void DBImpl::background_compaction() { } DBImpl::~DBImpl() { - std::unique_lock lock(_mutex); - _shutting_down.store(true, std::memory_order_release); - while (_bg_compaction_scheduled) { - _bg_work_finish_signal.wait(lock); + { + std::unique_lock lock(_mutex); + _shutting_down.store(true, std::memory_order_release); + while (_bg_compaction_scheduled) { + _bg_work_finish_signal.wait(lock); + } } + { + std::unique_lock lock(build_index_mutex_); + while (bg_build_index_started_) { + bg_build_index_finish_signal_.wait(lock); + } + } + std::vector ids; + _pMemMgr->serialize(ids); } /* diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index d678fc71d0..6ecc287dbe 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -64,7 +64,9 @@ private: Status _bg_error; std::atomic _shutting_down; + std::mutex build_index_mutex_; bool bg_build_index_started_; + std::condition_variable bg_build_index_finish_signal_; std::shared_ptr _pMeta; std::shared_ptr _pMemMgr;