From aa60c6287037f822344608f1232f8ba58f0d3d6a Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:03:13 +0800 Subject: [PATCH 01/31] refactor(db): remove dummy struct Former-commit-id: a90c3f127fb84a879956a8383af9def829504208 --- cpp/src/db/Options.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 100ab7b3ab..cd6f1f98b8 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -48,12 +48,6 @@ struct Options { }; // Options -struct GroupOptions { - size_t dimension; - bool has_id = false; -}; // GroupOptions - - } // namespace engine } // namespace vecwise } // namespace zilliz From e94499cdff237bd6242b7b272107b55ba458fd3c Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:16:16 +0800 Subject: [PATCH 02/31] refactor(db): change groupschema and groupfileschema to table* Former-commit-id: fbc69151149faa8ed32547bcdd1a9935dac34070 --- cpp/src/db/DB.h | 4 +- cpp/src/db/DBImpl.cpp | 22 ++-- cpp/src/db/DBImpl.h | 6 +- cpp/src/db/DBMetaImpl.cpp | 230 ++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 14 +- cpp/src/db/LocalMetaImpl.cpp | 28 ++-- cpp/src/db/LocalMetaImpl.h | 18 +-- cpp/src/db/MemManager.cpp | 8 +- cpp/src/db/MemManager.h | 4 +- cpp/src/db/Meta.h | 12 +- cpp/src/db/MetaTypes.h | 14 +- cpp/unittest/db/db_tests.cpp | 12 +- cpp/unittest/db/meta_tests.cpp | 46 +++---- 13 files changed, 209 insertions(+), 209 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 7d976ad824..036951e27b 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -21,8 +21,8 @@ class DB { public: static void Open(const Options& options, DB** dbptr); - virtual Status add_group(meta::GroupSchema& group_info_) = 0; - virtual Status get_group(meta::GroupSchema& group_info_) = 0; + virtual Status add_group(meta::TableSchema& group_info_) = 0; + virtual Status get_group(meta::TableSchema& group_info_) = 0; virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4028a1bbbe..c2eae4bfbf 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -35,12 +35,12 @@ DBImpl::DBImpl(const Options& options) } template -Status DBImpl::add_group(meta::GroupSchema& group_info) { +Status DBImpl::add_group(meta::TableSchema& group_info) { return _pMeta->add_group(group_info); } template -Status DBImpl::get_group(meta::GroupSchema& group_info) { +Status DBImpl::get_group(meta::TableSchema& group_info) { return _pMeta->get_group(group_info); } @@ -93,7 +93,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, meta::GroupFilesSchema raw_files; for (auto &day_files : files) { for (auto &file : day_files.second) { - file.file_type == meta::GroupFileSchema::INDEX ? + file.file_type == meta::TableFileSchema::INDEX ? index_files.push_back(file) : raw_files.push_back(file); } } @@ -259,7 +259,7 @@ void DBImpl::background_call() { template Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, const meta::GroupFilesSchema& files) { - meta::GroupFileSchema group_file; + meta::TableFileSchema group_file; group_file.group_id = group_id; group_file.date = date; Status status = _pMeta->add_group_file(group_file); @@ -277,7 +277,7 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat for (auto& file : files) { index.Merge(file.location); auto file_schema = file; - file_schema.file_type = meta::GroupFileSchema::TO_DELETE; + file_schema.file_type = meta::TableFileSchema::TO_DELETE; updated.push_back(file_schema); LOG(DEBUG) << "Merging file " << file_schema.file_id; index_size = index.Size(); @@ -288,9 +288,9 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat index.Serialize(); if (index_size >= _options.index_trigger_size) { - group_file.file_type = meta::GroupFileSchema::TO_INDEX; + group_file.file_type = meta::TableFileSchema::TO_INDEX; } else { - group_file.file_type = meta::GroupFileSchema::RAW; + group_file.file_type = meta::TableFileSchema::RAW; } group_file.size = index_size; updated.push_back(group_file); @@ -336,8 +336,8 @@ Status DBImpl::background_merge_files(const std::string& group_id) { } template -Status DBImpl::build_index(const meta::GroupFileSchema& file) { - meta::GroupFileSchema group_file; +Status DBImpl::build_index(const meta::TableFileSchema& file) { + meta::TableFileSchema group_file; group_file.group_id = file.group_id; group_file.date = file.date; Status status = _pMeta->add_group_file(group_file); @@ -350,11 +350,11 @@ Status DBImpl::build_index(const meta::GroupFileSchema& file) { to_index.Load(); auto index = to_index.BuildIndex(group_file.location); - group_file.file_type = meta::GroupFileSchema::INDEX; + group_file.file_type = meta::TableFileSchema::INDEX; group_file.size = index->Size(); auto to_remove = file; - to_remove.file_type = meta::GroupFileSchema::TO_DELETE; + to_remove.file_type = meta::TableFileSchema::TO_DELETE; meta::GroupFilesSchema update_files = {to_remove, group_file}; _pMeta->update_files(update_files); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 54c22eb48b..212b80dc9e 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -33,8 +33,8 @@ public: DBImpl(const Options& options); - virtual Status add_group(meta::GroupSchema& group_info) override; - virtual Status get_group(meta::GroupSchema& group_info) override; + virtual Status add_group(meta::TableSchema& group_info) override; + virtual Status get_group(meta::TableSchema& group_info) override; virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; @@ -62,7 +62,7 @@ public: private: void background_build_index(); - Status build_index(const meta::GroupFileSchema&); + Status build_index(const meta::TableFileSchema&); Status try_build_index(); Status merge_files(const std::string& group_id, const meta::DateT& date, diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 9b4e731f74..c467045e24 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -28,20 +28,20 @@ using namespace sqlite_orm; inline auto StoragePrototype(const std::string& path) { return make_storage(path, make_table("Group", - make_column("id", &GroupSchema::id, primary_key()), - make_column("group_id", &GroupSchema::group_id, unique()), - make_column("dimension", &GroupSchema::dimension), - make_column("created_on", &GroupSchema::created_on), - make_column("files_cnt", &GroupSchema::files_cnt, default_value(0))), + make_column("id", &TableSchema::id, primary_key()), + make_column("group_id", &TableSchema::group_id, unique()), + make_column("dimension", &TableSchema::dimension), + make_column("created_on", &TableSchema::created_on), + make_column("files_cnt", &TableSchema::files_cnt, default_value(0))), make_table("GroupFile", - make_column("id", &GroupFileSchema::id, primary_key()), - make_column("group_id", &GroupFileSchema::group_id), - make_column("file_id", &GroupFileSchema::file_id), - make_column("file_type", &GroupFileSchema::file_type), - make_column("size", &GroupFileSchema::size, default_value(0)), - make_column("updated_time", &GroupFileSchema::updated_time), - make_column("created_on", &GroupFileSchema::created_on), - make_column("date", &GroupFileSchema::date)) + make_column("id", &TableFileSchema::id, primary_key()), + make_column("group_id", &TableFileSchema::group_id), + make_column("file_id", &TableFileSchema::file_id), + make_column("file_type", &TableFileSchema::file_type), + make_column("size", &TableFileSchema::size, default_value(0)), + make_column("updated_time", &TableFileSchema::updated_time), + make_column("created_on", &TableFileSchema::created_on), + make_column("date", &TableFileSchema::date)) ); } @@ -59,7 +59,7 @@ std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, D return ss.str(); } -void DBMetaImpl::GetGroupFilePath(GroupFileSchema& group_file) { +void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) { if (group_file.date == EmptyDate) { group_file.date = Meta::GetDate(); } @@ -117,7 +117,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, return Status::OK(); } - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_id; auto status = get_group(group_info); if (!status.ok()) { @@ -135,11 +135,11 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, try { ConnectorPtr->update_all( set( - c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE + c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE ), where( - c(&GroupFileSchema::group_id) == group_id and - in(&GroupFileSchema::date, dates) + c(&TableFileSchema::group_id) == group_id and + in(&TableFileSchema::date, dates) )); } catch (std::exception & e) { LOG(DEBUG) << e.what(); @@ -148,7 +148,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, return Status::OK(); } -Status DBMetaImpl::add_group(GroupSchema& group_info) { +Status DBMetaImpl::add_group(TableSchema& group_info) { if (group_info.group_id == "") { NextGroupId(group_info.group_id); } @@ -178,17 +178,17 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) { return Status::OK(); } -Status DBMetaImpl::get_group(GroupSchema& group_info) { +Status DBMetaImpl::get_group(TableSchema& group_info) { return get_group_no_lock(group_info); } -Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) { +Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { try { - auto groups = ConnectorPtr->select(columns(&GroupSchema::id, - &GroupSchema::group_id, - &GroupSchema::files_cnt, - &GroupSchema::dimension), - where(c(&GroupSchema::group_id) == group_info.group_id)); + auto groups = ConnectorPtr->select(columns(&TableSchema::id, + &TableSchema::group_id, + &TableSchema::files_cnt, + &TableSchema::dimension), + where(c(&TableSchema::group_id) == group_info.group_id)); assert(groups.size() <= 1); if (groups.size() == 1) { group_info.id = std::get<0>(groups[0]); @@ -207,8 +207,8 @@ Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) { Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { try { - auto groups = ConnectorPtr->select(columns(&GroupSchema::id), - where(c(&GroupSchema::group_id) == group_id)); + auto groups = ConnectorPtr->select(columns(&TableSchema::id), + where(c(&TableSchema::group_id) == group_id)); assert(groups.size() <= 1); if (groups.size() == 1) { has_or_not = true; @@ -222,11 +222,11 @@ Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { return Status::OK(); } -Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { +Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { if (group_file.date == EmptyDate) { group_file.date = Meta::GetDate(); } - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_file.group_id; auto status = get_group(group_info); if (!status.ok()) { @@ -234,7 +234,7 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { } NextFileId(group_file.file_id); - group_file.file_type = GroupFileSchema::NEW; + group_file.file_type = TableFileSchema::NEW; group_file.dimension = group_info.dimension; group_file.size = 0; group_file.created_on = utils::GetMicroSecTimeStamp(); @@ -267,18 +267,18 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { files.clear(); try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX)); - std::map groups; + std::map groups; for (auto& file : selected) { - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.group_id = std::get<1>(file); group_file.file_id = std::get<2>(file); @@ -288,7 +288,7 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { GetGroupFilePath(group_file); auto groupItr = groups.find(group_file.group_id); if (groupItr == groups.end()) { - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_file.group_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { @@ -315,19 +315,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, const DatesT& dates = (partition.empty() == true) ? today : partition; try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::group_id) == group_id and - in(&GroupFileSchema::date, dates) and - (c(&GroupFileSchema::file_type) == (int) GroupFileSchema::RAW or - c(&GroupFileSchema::file_type) == (int) GroupFileSchema::TO_INDEX or - c(&GroupFileSchema::file_type) == (int) GroupFileSchema::INDEX))); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::group_id) == group_id and + in(&TableFileSchema::date, dates) and + (c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or + c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or + c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX))); - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { @@ -335,7 +335,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, } for (auto& file : selected) { - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.group_id = std::get<1>(file); group_file.file_id = std::get<2>(file); @@ -363,16 +363,16 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, files.clear(); try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW and - c(&GroupFileSchema::group_id) == group_id)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and + c(&TableFileSchema::group_id) == group_id)); - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { @@ -380,7 +380,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, } for (auto& file : selected) { - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.group_id = std::get<1>(file); group_file.file_id = std::get<2>(file); @@ -412,16 +412,16 @@ Status DBMetaImpl::has_group_file(const std::string& group_id_, Status DBMetaImpl::get_group_file(const std::string& group_id_, const std::string& file_id_, - GroupFileSchema& group_file_info_) { + TableFileSchema& group_file_info_) { try { - auto files = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::file_id) == file_id_ and - c(&GroupFileSchema::group_id) == group_id_ + auto files = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::file_id) == file_id_ and + c(&TableFileSchema::group_id) == group_id_ )); assert(files.size() <= 1); if (files.size() == 1) { @@ -466,11 +466,11 @@ Status DBMetaImpl::archive_files() { { ConnectorPtr->update_all( set( - c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE + c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE ), where( - c(&GroupFileSchema::created_on) < (long)(now - usecs) and - c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE + c(&TableFileSchema::created_on) < (long)(now - usecs) and + c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE )); } catch (std::exception & e) { LOG(DEBUG) << e.what(); @@ -493,9 +493,9 @@ Status DBMetaImpl::archive_files() { Status DBMetaImpl::size(long& result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&GroupFileSchema::size)), + auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)), where( - c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE + c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE )); for (auto& sub_query : selected) { @@ -518,16 +518,16 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { return Status::OK(); } try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::size), - where(c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE), - order_by(&GroupFileSchema::id), + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::size), + where(c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE), + order_by(&TableFileSchema::id), limit(10)); std::vector ids; for (auto& file : selected) { if (to_discard_size <= 0) break; - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.size = std::get<1>(file); ids.push_back(group_file.id); @@ -541,10 +541,10 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { ConnectorPtr->update_all( set( - c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE + c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE ), where( - in(&GroupFileSchema::id, ids) + in(&TableFileSchema::id, ids) )); } catch (std::exception & e) { @@ -556,7 +556,7 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { return discard_files_of_size(to_discard_size); } -Status DBMetaImpl::update_group_file(GroupFileSchema& group_file) { +Status DBMetaImpl::update_group_file(TableFileSchema& group_file) { group_file.updated_time = utils::GetMicroSecTimeStamp(); try { ConnectorPtr->update(group_file); @@ -590,19 +590,19 @@ Status DBMetaImpl::update_files(GroupFilesSchema& files) { Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE and - c(&GroupFileSchema::updated_time) > now - seconds*US_PS)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE and + c(&TableFileSchema::updated_time) > now - seconds*US_PS)); GroupFilesSchema updated; for (auto& file : selected) { - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.group_id = std::get<1>(file); group_file.file_id = std::get<2>(file); @@ -610,10 +610,10 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); GetGroupFilePath(group_file); - if (group_file.file_type == GroupFileSchema::TO_DELETE) { + if (group_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(group_file.location); } - ConnectorPtr->remove(group_file.id); + ConnectorPtr->remove(group_file.id); /* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */ } } catch (std::exception & e) { @@ -626,19 +626,19 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { Status DBMetaImpl::cleanup() { try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, - &GroupFileSchema::group_id, - &GroupFileSchema::file_id, - &GroupFileSchema::file_type, - &GroupFileSchema::size, - &GroupFileSchema::date), - where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE or - c(&GroupFileSchema::file_type) == (int)GroupFileSchema::NEW)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, + &TableFileSchema::group_id, + &TableFileSchema::file_id, + &TableFileSchema::file_type, + &TableFileSchema::size, + &TableFileSchema::date), + where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE or + c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW)); GroupFilesSchema updated; for (auto& file : selected) { - GroupFileSchema group_file; + TableFileSchema group_file; group_file.id = std::get<0>(file); group_file.group_id = std::get<1>(file); group_file.file_id = std::get<2>(file); @@ -646,10 +646,10 @@ Status DBMetaImpl::cleanup() { group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); GetGroupFilePath(group_file); - if (group_file.file_type == GroupFileSchema::TO_DELETE) { + if (group_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(group_file.location); } - ConnectorPtr->remove(group_file.id); + ConnectorPtr->remove(group_file.id); /* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */ } } catch (std::exception & e) { @@ -663,14 +663,14 @@ Status DBMetaImpl::cleanup() { Status DBMetaImpl::count(const std::string& group_id, long& result) { try { - auto selected = ConnectorPtr->select(columns(&GroupFileSchema::size, - &GroupFileSchema::date), - where((c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW or - c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX or - c(&GroupFileSchema::file_type) == (int)GroupFileSchema::INDEX) and - c(&GroupFileSchema::group_id) == group_id)); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, + &TableFileSchema::date), + where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or + c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or + c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and + c(&TableFileSchema::group_id) == group_id)); - GroupSchema group_info; + TableSchema group_info; group_info.group_id = group_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 6108860927..3b7a8f0981 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -19,11 +19,11 @@ class DBMetaImpl : public Meta { public: DBMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(GroupSchema& group_info) override; - virtual Status get_group(GroupSchema& group_info_) override; + virtual Status add_group(TableSchema& group_info) override; + virtual Status get_group(TableSchema& group_info_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; - virtual Status add_group_file(GroupFileSchema& group_file_info) override; + virtual Status add_group_file(TableFileSchema& group_file_info) override; virtual Status delete_group_partitions(const std::string& group_id, const meta::DatesT& dates) override; @@ -32,8 +32,8 @@ public: bool& has_or_not_) override; virtual Status get_group_file(const std::string& group_id_, const std::string& file_id_, - GroupFileSchema& group_file_info_) override; - virtual Status update_group_file(GroupFileSchema& group_file_) override; + TableFileSchema& group_file_info_) override; + virtual Status update_group_file(TableFileSchema& group_file_) override; virtual Status get_group_files(const std::string& group_id_, const int date_delta_, @@ -68,10 +68,10 @@ private: Status NextFileId(std::string& file_id); Status NextGroupId(std::string& group_id); Status discard_files_of_size(long to_discard_size); - Status get_group_no_lock(GroupSchema& group_info); + Status get_group_no_lock(TableSchema& group_info); std::string GetGroupPath(const std::string& group_id); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); - void GetGroupFilePath(GroupFileSchema& group_file); + void GetGroupFilePath(TableFileSchema& group_file); Status initialize(); const DBMetaOptions _options; diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index aa852a3db6..183f2e77e2 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id } std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, - GroupFileSchema::FILE_TYPE file_type) { - std::string suffix = (file_type == GroupFileSchema::RAW) ? ".raw" : ".index"; + TableFileSchema::FILE_TYPE file_type) { + std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index"; SimpleIDGenerator g; std::stringstream ss; ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; @@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) { return GetGroupMetaPathByGroupPath(GetGroupPath(group_id)); } -Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info) { +Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) { boost::property_tree::ptree ptree; boost::property_tree::read_json(path, ptree); auto files_cnt = ptree.get_child("files_cnt").data(); @@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem } -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info) { +Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) { group_info.group_id = group_id; return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info); } @@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() { return Status::OK(); } -Status LocalMetaImpl::add_group(GroupSchema& group_info) { +Status LocalMetaImpl::add_group(TableSchema& group_info) { std::string real_gid; size_t id = SimpleIDGenerator().getNextIDNumber(); if (group_info.group_id == "") { @@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) { return Status::OK(); } -Status LocalMetaImpl::get_group(GroupSchema& group_info) { +Status LocalMetaImpl::get_group(TableSchema& group_info) { bool group_exist; has_group(group_info.group_id, group_exist); if (!group_exist) { @@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { return Status::OK(); } -Status LocalMetaImpl::add_group_file(GroupFileSchema& group_file_info) { - GroupSchema group_info; +Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { + TableSchema group_info; /* auto status = get_group(group_info); */ /* if (!status.ok()) { */ /* return status; */ @@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { boost::filesystem::directory_iterator end_itr; for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { auto group_path = itr->path().string(); - GroupSchema group_info; + TableSchema group_info; GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { auto partition_path = innerItr->path().string(); @@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { if (suffix == "index") continue; if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue; std::cout << "[About to index] " << location << std::endl; - GroupFileSchema f; + TableFileSchema f; f.location = location; /* f.group_id = group_id; */ f.dimension = group_info.dimension; @@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, /* boost::filesystem::directory_iterator end_itr; */ /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ /* auto group_path = itr->path().string(); */ - /* GroupSchema group_info; */ + /* TableSchema group_info; */ /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */ /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ /* auto partition_path = innerItr->path().string(); */ @@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, /* if (suffix == "index") continue; */ /* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */ /* std::cout << "[About to index] " << location << std::endl; */ - /* GroupFileSchema f; */ + /* TableFileSchema f; */ /* f.location = location; */ /* f.group_id = group_id; */ /* f.dimension = group_info.dimension; */ @@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_, Status LocalMetaImpl::get_group_file(const std::string& group_id_, const std::string& file_id_, - GroupFileSchema& group_file_info_) { + TableFileSchema& group_file_info_) { //PXU TODO return Status::OK(); } @@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_, return Status::OK(); } -Status LocalMetaImpl::update_group_file(GroupFileSchema& group_file_) { +Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) { //PXU TODO return Status::OK(); } diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index fb989d5f67..76be0b7576 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -18,11 +18,11 @@ public: const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; LocalMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(GroupSchema& group_info_) override; - virtual Status get_group(GroupSchema& group_info_) override; + virtual Status add_group(TableSchema& group_info_) override; + virtual Status get_group(TableSchema& group_info_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; - virtual Status add_group_file(GroupFileSchema& group_file_info) override; + virtual Status add_group_file(TableFileSchema& group_file_info) override; /* virtual Status delete_group_partitions(const std::string& group_id, */ /* const meta::DatesT& dates) override; */ @@ -31,8 +31,8 @@ public: bool& has_or_not_) override; virtual Status get_group_file(const std::string& group_id_, const std::string& file_id_, - GroupFileSchema& group_file_info_) override; - virtual Status update_group_file(GroupFileSchema& group_file_) override; + TableFileSchema& group_file_info_) override; + virtual Status update_group_file(TableFileSchema& group_file_) override; virtual Status get_group_files(const std::string& group_id_, const int date_delta_, @@ -59,16 +59,16 @@ public: private: - Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info); + Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info); std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info); + Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info); std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, - GroupFileSchema::FILE_TYPE file_type); + TableFileSchema::FILE_TYPE file_type); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); std::string GetGroupPath(const std::string& group_id); std::string GetGroupMetaPath(const std::string& group_id); - Status CreateGroupMeta(const GroupSchema& group_schema); + Status CreateGroupMeta(const TableSchema& group_schema); long GetFileSize(const std::string& filename); Status initialize(); diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 601146ba66..8ba0d7dc72 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -22,7 +22,7 @@ namespace engine { template MemVectors::MemVectors(const std::shared_ptr& meta_ptr, - const meta::GroupFileSchema& schema, const Options& options) + const meta::TableFileSchema& schema, const Options& options) : pMeta_(meta_ptr), options_(options), schema_(schema), @@ -53,11 +53,11 @@ Status MemVectors::serialize(std::string& group_id) { pEE_->Serialize(); schema_.size = size; schema_.file_type = (size >= options_.index_trigger_size) ? - meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW; + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = pMeta_->update_group_file(schema_); - LOG(DEBUG) << "New " << ((schema_.file_type == meta::GroupFileSchema::RAW) ? "raw" : "to_index") + LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index") << " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M"; pEE_->Cache(); @@ -85,7 +85,7 @@ typename MemManager::MemVectorsPtr MemManager::get_mem_by_grou return memIt->second; } - meta::GroupFileSchema group_file; + meta::TableFileSchema group_file; group_file.group_id = group_id; auto status = _pMeta->add_group_file(group_file); if (!status.ok()) { diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 9b526dc1d9..a1b6e1c76a 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -31,7 +31,7 @@ public: typedef std::shared_ptr> Ptr; explicit MemVectors(const std::shared_ptr&, - const meta::GroupFileSchema&, const Options&); + const meta::TableFileSchema&, const Options&); void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); @@ -52,7 +52,7 @@ private: MetaPtr pMeta_; Options options_; - meta::GroupFileSchema schema_; + meta::TableFileSchema schema_; IDGenerator* _pIdGenerator; EnginePtr pEE_; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index a4bbb23380..3722109e02 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -20,13 +20,13 @@ namespace meta { class Meta { public: - typedef std::shared_ptr Ptr; + using Ptr = std::shared_ptr; - virtual Status add_group(GroupSchema& group_info) = 0; - virtual Status get_group(GroupSchema& group_info) = 0; + virtual Status add_group(TableSchema& group_info) = 0; + virtual Status get_group(TableSchema& group_info) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; - virtual Status add_group_file(GroupFileSchema& group_file_info) = 0; + virtual Status add_group_file(TableFileSchema& group_file_info) = 0; virtual Status delete_group_partitions(const std::string& group_id, const meta::DatesT& dates) = 0; @@ -35,8 +35,8 @@ public: bool& has_or_not_) = 0; virtual Status get_group_file(const std::string& group_id_, const std::string& file_id_, - GroupFileSchema& group_file_info_) = 0; - virtual Status update_group_file(GroupFileSchema& group_file_) = 0; + TableFileSchema& group_file_info_) = 0; + virtual Status update_group_file(TableFileSchema& group_file_) = 0; virtual Status get_group_files(const std::string& group_id_, const int date_delta_, diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index 4b956590ae..e48e97806c 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -18,16 +18,16 @@ typedef int DateT; const DateT EmptyDate = -1; typedef std::vector DatesT; -struct GroupSchema { +struct TableSchema { size_t id; std::string group_id; size_t files_cnt = 0; uint16_t dimension; - std::string location = ""; + std::string location; long created_on; -}; // GroupSchema +}; // TableSchema -struct GroupFileSchema { +struct TableFileSchema { typedef enum { NEW, RAW, @@ -43,12 +43,12 @@ struct GroupFileSchema { size_t size; DateT date = EmptyDate; uint16_t dimension; - std::string location = ""; + std::string location; long updated_time; long created_on; -}; // GroupFileSchema +}; // TableFileSchema -typedef std::vector GroupFilesSchema; +typedef std::vector GroupFilesSchema; typedef std::map DatePartionedGroupFilesSchema; } // namespace meta diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index f8ced228e0..c2d37a5e15 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { static const int group_dim = 256; long size; - engine::meta::GroupSchema group_info; + engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.group_id = group_name; engine::Status stat = db_->add_group(group_info); - engine::meta::GroupSchema group_info_get; + engine::meta::TableSchema group_info_get; group_info_get.group_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); @@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) { static const std::string group_name = "test_group"; static const int group_dim = 256; - engine::meta::GroupSchema group_info; + engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.group_id = group_name; engine::Status stat = db_->add_group(group_info); - engine::meta::GroupSchema group_info_get; + engine::meta::TableSchema group_info_get; group_info_get.group_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); @@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) { static const std::string group_name = "test_group"; static const int group_dim = 256; - engine::meta::GroupSchema group_info; + engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.group_id = group_name; engine::Status stat = db_->add_group(group_info); - engine::meta::GroupSchema group_info_get; + engine::meta::TableSchema group_info_get; group_info_get.group_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 3832a3263d..51bbd7957b 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine; TEST_F(MetaTest, GROUP_TEST) { auto group_id = "meta_test_group"; - meta::GroupSchema group; + meta::TableSchema group; group.group_id = group_id; auto status = impl_->add_group(group); ASSERT_TRUE(status.ok()); @@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) { TEST_F(MetaTest, GROUP_FILE_TEST) { auto group_id = "meta_test_group"; - meta::GroupSchema group; + meta::TableSchema group; group.group_id = group_id; auto status = impl_->add_group(group); - meta::GroupFileSchema group_file; + meta::TableFileSchema group_file; group_file.group_id = group.group_id; status = impl_->add_group_file(group_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.file_type, meta::GroupFileSchema::NEW); + ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW); auto file_id = group_file.file_id; - auto new_file_type = meta::GroupFileSchema::INDEX; + auto new_file_type = meta::TableFileSchema::INDEX; group_file.file_type = new_file_type; status = impl_->update_group_file(group_file); @@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { status = impl_->update_group_file(group_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2)); - ASSERT_FALSE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); + ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE); dates.clear(); dates.push_back(group_file.date); @@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { ASSERT_TRUE(status.ok()); status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); + ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { auto impl = meta::DBMetaImpl(options); auto group_id = "meta_test_group"; - meta::GroupSchema group; + meta::TableSchema group; group.group_id = group_id; auto status = impl.add_group(group); meta::GroupFilesSchema files; - meta::GroupFileSchema group_file; + meta::TableFileSchema group_file; group_file.group_id = group.group_id; auto cnt = 100; @@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { std::vector days; for (auto i=0; iadd_group(group); @@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { int to_index_files_cnt = 6; int index_files_cnt = 7; - meta::GroupFileSchema group_file; + meta::TableFileSchema group_file; group_file.group_id = group.group_id; for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::GroupFileSchema::NEW; + group_file.file_type = meta::TableFileSchema::NEW; status = impl_->update_group_file(group_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::GroupFileSchema::RAW; + group_file.file_type = meta::TableFileSchema::RAW; status = impl_->update_group_file(group_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::GroupFileSchema::TO_INDEX; + group_file.file_type = meta::TableFileSchema::TO_INDEX; status = impl_->update_group_file(group_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::GroupFileSchema::INDEX; + group_file.file_type = meta::TableFileSchema::INDEX; status = impl_->update_group_file(group_file); } From 209d0e480690746712083be44e83fa00284f2f83 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:20:20 +0800 Subject: [PATCH 03/31] refactor(db): change groupfilesschema to table* Former-commit-id: e786234155eecb9d22ad2b6a8b2c9357675e5de4 --- cpp/src/db/DB.h | 2 +- cpp/src/db/DBImpl.cpp | 20 ++++++++++---------- cpp/src/db/DBImpl.h | 4 ++-- cpp/src/db/DBMetaImpl.cpp | 18 +++++++++--------- cpp/src/db/DBMetaImpl.h | 10 +++++----- cpp/src/db/LocalMetaImpl.cpp | 8 ++++---- cpp/src/db/LocalMetaImpl.h | 8 ++++---- cpp/src/db/Meta.h | 10 +++++----- cpp/src/db/MetaTypes.h | 4 ++-- cpp/unittest/db/meta_tests.cpp | 8 ++++---- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 036951e27b..c1f73e5bed 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -28,7 +28,7 @@ public: virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status get_group_files(const std::string& group_id_, const int date_delta_, - meta::GroupFilesSchema& group_files_info_) = 0; + meta::TableFilesSchema& group_files_info_) = 0; virtual Status add_vectors(const std::string& group_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c2eae4bfbf..c3df297b28 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -58,7 +58,7 @@ Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not template Status DBImpl::get_group_files(const std::string& group_id, const int date_delta, - meta::GroupFilesSchema& group_files_info) { + meta::TableFilesSchema& group_files_info) { return _pMeta->get_group_files(group_id, date_delta, group_files_info); } @@ -83,14 +83,14 @@ template Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { - meta::DatePartionedGroupFilesSchema files; + meta::DatePartionedTableFilesSchema files; auto status = _pMeta->files_to_search(group_id, dates, files); if (!status.ok()) { return status; } LOG(DEBUG) << "Search DateT Size=" << files.size(); - meta::GroupFilesSchema index_files; - meta::GroupFilesSchema raw_files; + meta::TableFilesSchema index_files; + meta::TableFilesSchema raw_files; for (auto &day_files : files) { for (auto &file : day_files.second) { file.file_type == meta::TableFileSchema::INDEX ? @@ -132,7 +132,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, long search_set_size = 0; - auto search_in_index = [&](meta::GroupFilesSchema& file_vec) -> void { + auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { for (auto &file : file_vec) { EngineT index(file.dimension, file.location); index.Load(); @@ -258,7 +258,7 @@ void DBImpl::background_call() { template Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, - const meta::GroupFilesSchema& files) { + const meta::TableFilesSchema& files) { meta::TableFileSchema group_file; group_file.group_id = group_id; group_file.date = date; @@ -271,7 +271,7 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat EngineT index(group_file.dimension, group_file.location); - meta::GroupFilesSchema updated; + meta::TableFilesSchema updated; long index_size = 0; for (auto& file : files) { @@ -305,7 +305,7 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat template Status DBImpl::background_merge_files(const std::string& group_id) { - meta::DatePartionedGroupFilesSchema raw_files; + meta::DatePartionedTableFilesSchema raw_files; auto status = _pMeta->files_to_merge(group_id, raw_files); if (!status.ok()) { return status; @@ -356,7 +356,7 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { auto to_remove = file; to_remove.file_type = meta::TableFileSchema::TO_DELETE; - meta::GroupFilesSchema update_files = {to_remove, group_file}; + meta::TableFilesSchema update_files = {to_remove, group_file}; _pMeta->update_files(update_files); LOG(DEBUG) << "New index file " << group_file.file_id << " of size " @@ -373,7 +373,7 @@ template void DBImpl::background_build_index() { std::lock_guard lock(build_index_mutex_); assert(bg_build_index_started_); - meta::GroupFilesSchema to_index_files; + meta::TableFilesSchema to_index_files; _pMeta->files_to_index(to_index_files); Status status; for (auto& file : to_index_files) { diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 212b80dc9e..a727bc6e04 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -40,7 +40,7 @@ public: virtual Status get_group_files(const std::string& group_id_, const int date_delta_, - meta::GroupFilesSchema& group_files_info_) override; + meta::TableFilesSchema& group_files_info_) override; virtual Status add_vectors(const std::string& group_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) override; @@ -66,7 +66,7 @@ private: Status try_build_index(); Status merge_files(const std::string& group_id, const meta::DateT& date, - const meta::GroupFilesSchema& files); + const meta::TableFilesSchema& files); Status background_merge_files(const std::string& group_id); void try_schedule_compaction(); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index c467045e24..15c3a5eda0 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -263,7 +263,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { return Status::OK(); } -Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { +Status DBMetaImpl::files_to_index(TableFilesSchema& files) { files.clear(); try { @@ -309,7 +309,7 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { Status DBMetaImpl::files_to_search(const std::string &group_id, const DatesT& partition, - DatePartionedGroupFilesSchema &files) { + DatePartionedTableFilesSchema &files) { files.clear(); DatesT today = {Meta::GetDate()}; const DatesT& dates = (partition.empty() == true) ? today : partition; @@ -346,7 +346,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { - files[group_file.date] = GroupFilesSchema(); + files[group_file.date] = TableFilesSchema(); } files[group_file.date].push_back(group_file); } @@ -359,7 +359,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, } Status DBMetaImpl::files_to_merge(const std::string& group_id, - DatePartionedGroupFilesSchema& files) { + DatePartionedTableFilesSchema& files) { files.clear(); try { @@ -391,7 +391,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { - files[group_file.date] = GroupFilesSchema(); + files[group_file.date] = TableFilesSchema(); } files[group_file.date].push_back(group_file); } @@ -444,7 +444,7 @@ Status DBMetaImpl::get_group_file(const std::string& group_id_, Status DBMetaImpl::get_group_files(const std::string& group_id_, const int date_delta_, - GroupFilesSchema& group_files_info_) { + TableFilesSchema& group_files_info_) { // PXU TODO return Status::OK(); } @@ -568,7 +568,7 @@ Status DBMetaImpl::update_group_file(TableFileSchema& group_file) { return Status::OK(); } -Status DBMetaImpl::update_files(GroupFilesSchema& files) { +Status DBMetaImpl::update_files(TableFilesSchema& files) { try { auto commited = ConnectorPtr->transaction([&] () mutable { for (auto& file : files) { @@ -599,7 +599,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE and c(&TableFileSchema::updated_time) > now - seconds*US_PS)); - GroupFilesSchema updated; + TableFilesSchema updated; for (auto& file : selected) { TableFileSchema group_file; @@ -635,7 +635,7 @@ Status DBMetaImpl::cleanup() { where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE or c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW)); - GroupFilesSchema updated; + TableFilesSchema updated; for (auto& file : selected) { TableFileSchema group_file; diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 3b7a8f0981..1479396e84 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -37,18 +37,18 @@ public: virtual Status get_group_files(const std::string& group_id_, const int date_delta_, - GroupFilesSchema& group_files_info_) override; + TableFilesSchema& group_files_info_) override; - virtual Status update_files(GroupFilesSchema& files) override; + virtual Status update_files(TableFilesSchema& files) override; virtual Status files_to_merge(const std::string& group_id, - DatePartionedGroupFilesSchema& files) override; + DatePartionedTableFilesSchema& files) override; virtual Status files_to_search(const std::string& group_id, const DatesT& partition, - DatePartionedGroupFilesSchema& files) override; + DatePartionedTableFilesSchema& files) override; - virtual Status files_to_index(GroupFilesSchema&) override; + virtual Status files_to_index(TableFilesSchema&) override; virtual Status archive_files() override; diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index 183f2e77e2..f254d238de 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -152,7 +152,7 @@ Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { return Status::OK(); } -Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { +Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { files.clear(); std::string suffix; @@ -182,7 +182,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) { } Status LocalMetaImpl::files_to_merge(const std::string& group_id, - DatePartionedGroupFilesSchema& files) { + DatePartionedTableFilesSchema& files) { files.clear(); /* std::string suffix; */ /* boost::filesystem::directory_iterator end_itr; */ @@ -226,7 +226,7 @@ Status LocalMetaImpl::get_group_file(const std::string& group_id_, Status LocalMetaImpl::get_group_files(const std::string& group_id_, const int date_delta_, - GroupFilesSchema& group_files_info_) { + TableFilesSchema& group_files_info_) { // PXU TODO return Status::OK(); } @@ -236,7 +236,7 @@ Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) { return Status::OK(); } -Status LocalMetaImpl::update_files(GroupFilesSchema& files) { +Status LocalMetaImpl::update_files(TableFilesSchema& files) { //PXU TODO return Status::OK(); } diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 76be0b7576..952e85b1a3 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -36,16 +36,16 @@ public: virtual Status get_group_files(const std::string& group_id_, const int date_delta_, - GroupFilesSchema& group_files_info_) override; + TableFilesSchema& group_files_info_) override; - virtual Status update_files(GroupFilesSchema& files) override; + virtual Status update_files(TableFilesSchema& files) override; virtual Status cleanup() override; virtual Status files_to_merge(const std::string& group_id, - DatePartionedGroupFilesSchema& files) override; + DatePartionedTableFilesSchema& files) override; - virtual Status files_to_index(GroupFilesSchema&) override; + virtual Status files_to_index(TableFilesSchema&) override; virtual Status archive_files() override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 3722109e02..afa9eee2d7 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -40,22 +40,22 @@ public: virtual Status get_group_files(const std::string& group_id_, const int date_delta_, - GroupFilesSchema& group_files_info_) = 0; + TableFilesSchema& group_files_info_) = 0; - virtual Status update_files(GroupFilesSchema& files) = 0; + virtual Status update_files(TableFilesSchema& files) = 0; virtual Status files_to_search(const std::string& group_id, const DatesT& partition, - DatePartionedGroupFilesSchema& files) = 0; + DatePartionedTableFilesSchema& files) = 0; virtual Status files_to_merge(const std::string& group_id, - DatePartionedGroupFilesSchema& files) = 0; + DatePartionedTableFilesSchema& files) = 0; virtual Status size(long& result) = 0; virtual Status archive_files() = 0; - virtual Status files_to_index(GroupFilesSchema&) = 0; + virtual Status files_to_index(TableFilesSchema&) = 0; virtual Status cleanup() = 0; virtual Status cleanup_ttl_files(uint16_t) = 0; diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index e48e97806c..88bb112de9 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -48,8 +48,8 @@ struct TableFileSchema { long created_on; }; // TableFileSchema -typedef std::vector GroupFilesSchema; -typedef std::map DatePartionedGroupFilesSchema; +typedef std::vector TableFilesSchema; +typedef std::map DatePartionedTableFilesSchema; } // namespace meta } // namespace engine diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 51bbd7957b..62b1fce1b0 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -106,7 +106,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { group.group_id = group_id; auto status = impl.add_group(group); - meta::GroupFilesSchema files; + meta::TableFilesSchema files; meta::TableFileSchema group_file; group_file.group_id = group.group_id; @@ -152,7 +152,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { group.group_id = group_id; auto status = impl.add_group(group); - meta::GroupFilesSchema files; + meta::TableFilesSchema files; meta::TableFileSchema group_file; group_file.group_id = group.group_id; @@ -222,13 +222,13 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { status = impl_->update_group_file(group_file); } - meta::GroupFilesSchema files; + meta::TableFilesSchema files; status = impl_->files_to_index(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatePartionedGroupFilesSchema dated_files; + meta::DatePartionedTableFilesSchema dated_files; status = impl_->files_to_merge(group.group_id, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt); From f2bb20c90d4c27f328180c9aa02c35735bcca736 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:30:17 +0800 Subject: [PATCH 04/31] refactor(db): change group_id to table_id Former-commit-id: bf0cec2552fcad95bf1b77cdfbdd423984e6f18f --- cpp/src/db/DB.h | 14 ++--- cpp/src/db/DBImpl.cpp | 50 ++++++++-------- cpp/src/db/DBImpl.h | 18 +++--- cpp/src/db/DBMetaImpl.cpp | 102 ++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 22 +++---- cpp/src/db/LocalMetaImpl.cpp | 60 +++++++++---------- cpp/src/db/LocalMetaImpl.h | 24 ++++---- cpp/src/db/MemManager.cpp | 34 +++++------ cpp/src/db/MemManager.h | 10 ++-- cpp/src/db/Meta.h | 16 +++--- cpp/src/db/MetaTypes.h | 4 +- cpp/src/db/db_connection.cpp | 31 ---------- cpp/src/db/db_connection.h | 64 --------------------- cpp/unittest/db/db_tests.cpp | 12 ++-- cpp/unittest/db/meta_tests.cpp | 50 ++++++++-------- 15 files changed, 208 insertions(+), 303 deletions(-) delete mode 100644 cpp/src/db/db_connection.cpp delete mode 100644 cpp/src/db/db_connection.h diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index c1f73e5bed..20e4924076 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -23,27 +23,27 @@ public: virtual Status add_group(meta::TableSchema& group_info_) = 0; virtual Status get_group(meta::TableSchema& group_info_) = 0; - virtual Status delete_vectors(const std::string& group_id, + virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; - virtual Status get_group_files(const std::string& group_id_, + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, meta::TableFilesSchema& group_files_info_) = 0; - virtual Status add_vectors(const std::string& group_id_, + virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) = 0; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; virtual Status size(long& result) = 0; virtual Status drop_all() = 0; - virtual Status count(const std::string& group_id, long& result) = 0; + virtual Status count(const std::string& table_id, long& result) = 0; DB() = default; DB(const DB&) = delete; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c3df297b28..54a1380031 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -45,46 +45,46 @@ Status DBImpl::get_group(meta::TableSchema& group_info) { } template -Status DBImpl::delete_vectors(const std::string& group_id, +Status DBImpl::delete_vectors(const std::string& table_id, const meta::DatesT& dates) { - return _pMeta->delete_group_partitions(group_id, dates); + return _pMeta->delete_group_partitions(table_id, dates); } template -Status DBImpl::has_group(const std::string& group_id_, bool& has_or_not_) { - return _pMeta->has_group(group_id_, has_or_not_); +Status DBImpl::has_group(const std::string& table_id_, bool& has_or_not_) { + return _pMeta->has_group(table_id_, has_or_not_); } template -Status DBImpl::get_group_files(const std::string& group_id, +Status DBImpl::get_group_files(const std::string& table_id, const int date_delta, meta::TableFilesSchema& group_files_info) { - return _pMeta->get_group_files(group_id, date_delta, group_files_info); + return _pMeta->get_group_files(table_id, date_delta, group_files_info); } template -Status DBImpl::add_vectors(const std::string& group_id_, +Status DBImpl::add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { - Status status = _pMemMgr->add_vectors(group_id_, n, vectors, vector_ids_); + Status status = _pMemMgr->add_vectors(table_id_, n, vectors, vector_ids_); if (!status.ok()) { return status; } } template -Status DBImpl::search(const std::string &group_id, size_t k, size_t nq, +Status DBImpl::search(const std::string &table_id, size_t k, size_t nq, const float *vectors, QueryResults &results) { meta::DatesT dates = {meta::Meta::GetDate()}; - return search(group_id, k, nq, vectors, dates, results); + return search(table_id, k, nq, vectors, dates, results); } template -Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, +Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; - auto status = _pMeta->files_to_search(group_id, dates, files); + auto status = _pMeta->files_to_search(table_id, dates, files); if (!status.ok()) { return status; } LOG(DEBUG) << "Search DateT Size=" << files.size(); @@ -204,7 +204,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, } if (results.empty()) { - return Status::NotFound("Group " + group_id + ", search result not found!"); + return Status::NotFound("Group " + table_id + ", search result not found!"); } return Status::OK(); } @@ -257,10 +257,10 @@ void DBImpl::background_call() { template -Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date, +Status DBImpl::merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { meta::TableFileSchema group_file; - group_file.group_id = group_id; + group_file.table_id = table_id; group_file.date = date; Status status = _pMeta->add_group_file(group_file); @@ -304,9 +304,9 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::Dat } template -Status DBImpl::background_merge_files(const std::string& group_id) { +Status DBImpl::background_merge_files(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; - auto status = _pMeta->files_to_merge(group_id, raw_files); + auto status = _pMeta->files_to_merge(table_id, raw_files); if (!status.ok()) { return status; } @@ -323,7 +323,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) { continue; } has_merge = true; - merge_files(group_id, kv.first, kv.second); + merge_files(table_id, kv.first, kv.second); } _pMeta->archive_files(); @@ -338,7 +338,7 @@ Status DBImpl::background_merge_files(const std::string& group_id) { template Status DBImpl::build_index(const meta::TableFileSchema& file) { meta::TableFileSchema group_file; - group_file.group_id = file.group_id; + group_file.table_id = file.table_id; group_file.date = file.date; Status status = _pMeta->add_group_file(group_file); if (!status.ok()) { @@ -402,12 +402,12 @@ Status DBImpl::try_build_index() { template void DBImpl::background_compaction() { - std::vector group_ids; - _pMemMgr->serialize(group_ids); + std::vector table_ids; + _pMemMgr->serialize(table_ids); Status status; - for (auto group_id : group_ids) { - status = background_merge_files(group_id); + for (auto table_id : table_ids) { + status = background_merge_files(table_id); if (!status.ok()) { _bg_error = status; return; @@ -421,8 +421,8 @@ Status DBImpl::drop_all() { } template -Status DBImpl::count(const std::string& group_id, long& result) { - return _pMeta->count(group_id, result); +Status DBImpl::count(const std::string& table_id, long& result) { + return _pMeta->count(table_id, result); } template diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index a727bc6e04..35a4a93eff 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -35,25 +35,25 @@ public: virtual Status add_group(meta::TableSchema& group_info) override; virtual Status get_group(meta::TableSchema& group_info) override; - virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, meta::TableFilesSchema& group_files_info_) override; - virtual Status add_vectors(const std::string& group_id_, + virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) override; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) override; - virtual Status search(const std::string& group_id, size_t k, size_t nq, + virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) override; virtual Status drop_all() override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual Status size(long& result) override; @@ -64,10 +64,10 @@ private: void background_build_index(); Status build_index(const meta::TableFileSchema&); Status try_build_index(); - Status merge_files(const std::string& group_id, + Status merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files); - Status background_merge_files(const std::string& group_id); + Status background_merge_files(const std::string& table_id); void try_schedule_compaction(); void start_timer_task(int interval_); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 15c3a5eda0..41fceb5c0f 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -29,13 +29,13 @@ inline auto StoragePrototype(const std::string& path) { return make_storage(path, make_table("Group", make_column("id", &TableSchema::id, primary_key()), - make_column("group_id", &TableSchema::group_id, unique()), + make_column("table_id", &TableSchema::table_id, unique()), make_column("dimension", &TableSchema::dimension), make_column("created_on", &TableSchema::created_on), make_column("files_cnt", &TableSchema::files_cnt, default_value(0))), make_table("GroupFile", make_column("id", &TableFileSchema::id, primary_key()), - make_column("group_id", &TableFileSchema::group_id), + make_column("table_id", &TableFileSchema::table_id), make_column("file_id", &TableFileSchema::file_id), make_column("file_type", &TableFileSchema::file_type), make_column("size", &TableFileSchema::size, default_value(0)), @@ -49,13 +49,13 @@ inline auto StoragePrototype(const std::string& path) { using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -std::string DBMetaImpl::GetGroupPath(const std::string& group_id) { - return _options.path + "/tables/" + group_id; +std::string DBMetaImpl::GetGroupPath(const std::string& table_id) { + return _options.path + "/tables/" + table_id; } -std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) { +std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date; + ss << GetGroupPath(table_id) << "/" << date; return ss.str(); } @@ -64,16 +64,16 @@ void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) { group_file.date = Meta::GetDate(); } std::stringstream ss; - ss << GetGroupDatePartitionPath(group_file.group_id, group_file.date) + ss << GetGroupDatePartitionPath(group_file.table_id, group_file.date) << "/" << group_file.file_id; group_file.location = ss.str(); } -Status DBMetaImpl::NextGroupId(std::string& group_id) { +Status DBMetaImpl::NextGroupId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.getNextIDNumber(); - group_id = ss.str(); + table_id = ss.str(); return Status::OK(); } @@ -111,14 +111,14 @@ Status DBMetaImpl::initialize() { } // PXU TODO: Temp solution. Will fix later -Status DBMetaImpl::delete_group_partitions(const std::string& group_id, +Status DBMetaImpl::delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) { if (dates.size() == 0) { return Status::OK(); } TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group(group_info); if (!status.ok()) { return status; @@ -138,7 +138,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE ), where( - c(&TableFileSchema::group_id) == group_id and + c(&TableFileSchema::table_id) == table_id and in(&TableFileSchema::date, dates) )); } catch (std::exception & e) { @@ -149,8 +149,8 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id, } Status DBMetaImpl::add_group(TableSchema& group_info) { - if (group_info.group_id == "") { - NextGroupId(group_info.group_id); + if (group_info.table_id == "") { + NextGroupId(group_info.table_id); } group_info.files_cnt = 0; group_info.id = -1; @@ -165,7 +165,7 @@ Status DBMetaImpl::add_group(TableSchema& group_info) { } } - auto group_path = GetGroupPath(group_info.group_id); + auto group_path = GetGroupPath(group_info.table_id); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -185,17 +185,17 @@ Status DBMetaImpl::get_group(TableSchema& group_info) { Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id, - &TableSchema::group_id, + &TableSchema::table_id, &TableSchema::files_cnt, &TableSchema::dimension), - where(c(&TableSchema::group_id) == group_info.group_id)); + where(c(&TableSchema::table_id) == group_info.table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { group_info.id = std::get<0>(groups[0]); group_info.files_cnt = std::get<2>(groups[0]); group_info.dimension = std::get<3>(groups[0]); } else { - return Status::NotFound("Group " + group_info.group_id + " not found"); + return Status::NotFound("Group " + group_info.table_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -205,10 +205,10 @@ Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { return Status::OK(); } -Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { +Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id), - where(c(&TableSchema::group_id) == group_id)); + where(c(&TableSchema::table_id) == table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { has_or_not = true; @@ -227,7 +227,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { group_file.date = Meta::GetDate(); } TableSchema group_info; - group_info.group_id = group_file.group_id; + group_info.table_id = group_file.table_id; auto status = get_group(group_info); if (!status.ok()) { return status; @@ -250,7 +250,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { } } - auto partition_path = GetGroupDatePartitionPath(group_file.group_id, group_file.date); + auto partition_path = GetGroupDatePartitionPath(group_file.table_id, group_file.date); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); @@ -268,7 +268,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -280,23 +280,23 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); GetGroupFilePath(group_file); - auto groupItr = groups.find(group_file.group_id); + auto groupItr = groups.find(group_file.table_id); if (groupItr == groups.end()) { TableSchema group_info; - group_info.group_id = group_file.group_id; + group_info.table_id = group_file.table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; } - groups[group_file.group_id] = group_info; + groups[group_file.table_id] = group_info; } - group_file.dimension = groups[group_file.group_id].dimension; + group_file.dimension = groups[group_file.table_id].dimension; files.push_back(group_file); } } catch (std::exception & e) { @@ -307,7 +307,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { return Status::OK(); } -Status DBMetaImpl::files_to_search(const std::string &group_id, +Status DBMetaImpl::files_to_search(const std::string &table_id, const DatesT& partition, DatePartionedTableFilesSchema &files) { files.clear(); @@ -316,19 +316,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), - where(c(&TableFileSchema::group_id) == group_id and + where(c(&TableFileSchema::table_id) == table_id and in(&TableFileSchema::date, dates) and (c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX))); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; @@ -337,7 +337,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -358,22 +358,22 @@ Status DBMetaImpl::files_to_search(const std::string &group_id, return Status::OK(); } -Status DBMetaImpl::files_to_merge(const std::string& group_id, +Status DBMetaImpl::files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and - c(&TableFileSchema::group_id) == group_id)); + c(&TableFileSchema::table_id) == table_id)); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; @@ -382,7 +382,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -403,30 +403,30 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, return Status::OK(); } -Status DBMetaImpl::has_group_file(const std::string& group_id_, +Status DBMetaImpl::has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) { //PXU TODO return Status::OK(); } -Status DBMetaImpl::get_group_file(const std::string& group_id_, +Status DBMetaImpl::get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) { try { auto files = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), where(c(&TableFileSchema::file_id) == file_id_ and - c(&TableFileSchema::group_id) == group_id_ + c(&TableFileSchema::table_id) == table_id_ )); assert(files.size() <= 1); if (files.size() == 1) { group_file_info_.id = std::get<0>(files[0]); - group_file_info_.group_id = std::get<1>(files[0]); + group_file_info_.table_id = std::get<1>(files[0]); group_file_info_.file_id = std::get<2>(files[0]); group_file_info_.file_type = std::get<3>(files[0]); group_file_info_.size = std::get<4>(files[0]); @@ -442,7 +442,7 @@ Status DBMetaImpl::get_group_file(const std::string& group_id_, return Status::OK(); } -Status DBMetaImpl::get_group_files(const std::string& group_id_, +Status DBMetaImpl::get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) { // PXU TODO @@ -591,7 +591,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -604,7 +604,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -627,7 +627,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { Status DBMetaImpl::cleanup() { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::group_id, + &TableFileSchema::table_id, &TableFileSchema::file_id, &TableFileSchema::file_type, &TableFileSchema::size, @@ -640,7 +640,7 @@ Status DBMetaImpl::cleanup() { for (auto& file : selected) { TableFileSchema group_file; group_file.id = std::get<0>(file); - group_file.group_id = std::get<1>(file); + group_file.table_id = std::get<1>(file); group_file.file_id = std::get<2>(file); group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); @@ -660,7 +660,7 @@ Status DBMetaImpl::cleanup() { return Status::OK(); } -Status DBMetaImpl::count(const std::string& group_id, long& result) { +Status DBMetaImpl::count(const std::string& table_id, long& result) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, @@ -668,10 +668,10 @@ Status DBMetaImpl::count(const std::string& group_id, long& result) { where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and - c(&TableFileSchema::group_id) == group_id)); + c(&TableFileSchema::table_id) == table_id)); TableSchema group_info; - group_info.group_id = group_id; + group_info.table_id = table_id; auto status = get_group_no_lock(group_info); if (!status.ok()) { return status; diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 1479396e84..b278bee268 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -21,30 +21,30 @@ public: virtual Status add_group(TableSchema& group_info) override; virtual Status get_group(TableSchema& group_info_) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; - virtual Status delete_group_partitions(const std::string& group_id, + virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) override; - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) override; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) override; virtual Status update_group_file(TableFileSchema& group_file_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) override; virtual Status update_files(TableFilesSchema& files) override; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - virtual Status files_to_search(const std::string& group_id, + virtual Status files_to_search(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) override; @@ -60,17 +60,17 @@ public: virtual Status drop_all() override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual ~DBMetaImpl(); private: Status NextFileId(std::string& file_id); - Status NextGroupId(std::string& group_id); + Status NextGroupId(std::string& table_id); Status discard_files_of_size(long to_discard_size); Status get_group_no_lock(TableSchema& group_info); - std::string GetGroupPath(const std::string& group_id); - std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); + std::string GetGroupPath(const std::string& table_id); + std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); void GetGroupFilePath(TableFileSchema& group_file); Status initialize(); diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index f254d238de..70a073506d 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -27,22 +27,22 @@ long LocalMetaImpl::GetFileSize(const std::string& filename) return rc == 0 ? stat_buf.st_size : -1; } -std::string LocalMetaImpl::GetGroupPath(const std::string& group_id) { - return _options.path + "/" + group_id; +std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) { + return _options.path + "/" + table_id; } -std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) { +std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date; + ss << GetGroupPath(table_id) << "/" << date; return ss.str(); } -std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, +std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type) { std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index"; SimpleIDGenerator g; std::stringstream ss; - ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; + ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; return ss.str(); } @@ -50,8 +50,8 @@ std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_ return group_path + "/" + "meta"; } -std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) { - return GetGroupMetaPathByGroupPath(GetGroupPath(group_id)); +std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) { + return GetGroupMetaPathByGroupPath(GetGroupPath(table_id)); } Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) { @@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem /* std::cout << dimension << std::endl; */ /* std::cout << files_cnt << std::endl; */ - group_info.id = std::stoi(group_info.group_id); + group_info.id = std::stoi(group_info.table_id); group_info.files_cnt = std::stoi(files_cnt); group_info.dimension = std::stoi(dimension); - group_info.location = GetGroupPath(group_info.group_id); + group_info.location = GetGroupPath(group_info.table_id); return Status::OK(); } -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) { - group_info.group_id = group_id; - return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info); +Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) { + group_info.table_id = table_id; + return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info); } LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_) @@ -93,12 +93,12 @@ Status LocalMetaImpl::initialize() { Status LocalMetaImpl::add_group(TableSchema& group_info) { std::string real_gid; size_t id = SimpleIDGenerator().getNextIDNumber(); - if (group_info.group_id == "") { + if (group_info.table_id == "") { std::stringstream ss; ss << id; real_gid = ss.str(); } else { - real_gid = group_info.group_id; + real_gid = group_info.table_id; } bool group_exist; @@ -110,7 +110,7 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { return Status::GroupError("Cannot Create Group " + real_gid); } - group_info.group_id = real_gid; + group_info.table_id = real_gid; group_info.files_cnt = 0; group_info.id = 0; group_info.location = GetGroupPath(real_gid); @@ -125,16 +125,16 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { Status LocalMetaImpl::get_group(TableSchema& group_info) { bool group_exist; - has_group(group_info.group_id, group_exist); + has_group(group_info.table_id, group_exist); if (!group_exist) { - return Status::NotFound("Group " + group_info.group_id + " Not Found"); + return Status::NotFound("Group " + group_info.table_id + " Not Found"); } - return GetGroupMetaInfo(group_info.group_id, group_info); + return GetGroupMetaInfo(group_info.table_id, group_info); } -Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { - has_or_not = boost::filesystem::is_directory(GetGroupPath(group_id)); +Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { + has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id)); return Status::OK(); } @@ -144,8 +144,8 @@ Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { /* if (!status.ok()) { */ /* return status; */ /* } */ - /* auto location = GetNextGroupFileLocationByPartition(group_id, date, file_type); */ - /* group_file_info.group_id = group_id; */ + /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */ + /* group_file_info.table_id = table_id; */ /* group_file_info.dimension = group_info.dimension; */ /* group_file_info.location = location; */ /* group_file_info.date = date; */ @@ -171,7 +171,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { std::cout << "[About to index] " << location << std::endl; TableFileSchema f; f.location = location; - /* f.group_id = group_id; */ + /* f.table_id = table_id; */ f.dimension = group_info.dimension; files.push_back(f); } @@ -181,7 +181,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { return Status::OK(); } -Status LocalMetaImpl::files_to_merge(const std::string& group_id, +Status LocalMetaImpl::files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); /* std::string suffix; */ @@ -200,7 +200,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, /* std::cout << "[About to index] " << location << std::endl; */ /* TableFileSchema f; */ /* f.location = location; */ - /* f.group_id = group_id; */ + /* f.table_id = table_id; */ /* f.dimension = group_info.dimension; */ /* files.push_back(f); */ /* } */ @@ -210,21 +210,21 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id, return Status::OK(); } -Status LocalMetaImpl::has_group_file(const std::string& group_id_, +Status LocalMetaImpl::has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) { //PXU TODO return Status::OK(); } -Status LocalMetaImpl::get_group_file(const std::string& group_id_, +Status LocalMetaImpl::get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) { //PXU TODO return Status::OK(); } -Status LocalMetaImpl::get_group_files(const std::string& group_id_, +Status LocalMetaImpl::get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) { // PXU TODO @@ -266,7 +266,7 @@ Status LocalMetaImpl::size(long& result) { return Status::OK(); } -Status LocalMetaImpl::count(const std::string& group_id, long& result) { +Status LocalMetaImpl::count(const std::string& table_id, long& result) { // PXU TODO return Status::OK(); } diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 952e85b1a3..e57f0a97c1 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -20,21 +20,21 @@ public: virtual Status add_group(TableSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; - /* virtual Status delete_group_partitions(const std::string& group_id, */ + /* virtual Status delete_group_partitions(const std::string& table_id, */ /* const meta::DatesT& dates) override; */ - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) override; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) override; virtual Status update_group_file(TableFileSchema& group_file_) override; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) override; @@ -42,7 +42,7 @@ public: virtual Status cleanup() override; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; virtual Status files_to_index(TableFilesSchema&) override; @@ -51,7 +51,7 @@ public: virtual Status cleanup_ttl_files(uint16_t seconds) override; - virtual Status count(const std::string& group_id, long& result) override; + virtual Status count(const std::string& table_id, long& result) override; virtual Status drop_all() override; @@ -61,12 +61,12 @@ private: Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info); std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info); - std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, + Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info); + std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type); - std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); - std::string GetGroupPath(const std::string& group_id); - std::string GetGroupMetaPath(const std::string& group_id); + std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); + std::string GetGroupPath(const std::string& table_id); + std::string GetGroupMetaPath(const std::string& table_id); Status CreateGroupMeta(const TableSchema& group_schema); long GetFileSize(const std::string& filename); diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 8ba0d7dc72..bd69bbc2a8 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -47,8 +47,8 @@ size_t MemVectors::approximate_size() const { } template -Status MemVectors::serialize(std::string& group_id) { - group_id = schema_.group_id; +Status MemVectors::serialize(std::string& table_id) { + table_id = schema_.table_id; auto size = approximate_size(); pEE_->Serialize(); schema_.size = size; @@ -79,40 +79,40 @@ MemVectors::~MemVectors() { template typename MemManager::MemVectorsPtr MemManager::get_mem_by_group( - const std::string& group_id) { - auto memIt = _memMap.find(group_id); + const std::string& table_id) { + auto memIt = _memMap.find(table_id); if (memIt != _memMap.end()) { return memIt->second; } meta::TableFileSchema group_file; - group_file.group_id = group_id; + group_file.table_id = table_id; auto status = _pMeta->add_group_file(group_file); if (!status.ok()) { return nullptr; } - _memMap[group_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); - return _memMap[group_id]; + _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); + return _memMap[table_id]; } template -Status MemManager::add_vectors(const std::string& group_id_, +Status MemManager::add_vectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { std::unique_lock lock(_mutex); - return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); + return add_vectors_no_lock(table_id_, n_, vectors_, vector_ids_); } template -Status MemManager::add_vectors_no_lock(const std::string& group_id, +Status MemManager::add_vectors_no_lock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { - MemVectorsPtr mem = get_mem_by_group(group_id); + MemVectorsPtr mem = get_mem_by_group(table_id); if (mem == nullptr) { - return Status::NotFound("Group " + group_id + " not found!"); + return Status::NotFound("Group " + table_id + " not found!"); } mem->add(n, vectors, vector_ids); @@ -131,14 +131,14 @@ Status MemManager::mark_memory_as_immutable() { } template -Status MemManager::serialize(std::vector& group_ids) { +Status MemManager::serialize(std::vector& table_ids) { mark_memory_as_immutable(); std::unique_lock lock(serialization_mtx_); - std::string group_id; - group_ids.clear(); + std::string table_id; + table_ids.clear(); for (auto& mem : _immMems) { - mem->serialize(group_id); - group_ids.push_back(group_id); + mem->serialize(table_id); + table_ids.push_back(table_id); } _immMems.clear(); return Status::OK(); diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index a1b6e1c76a..b6e4b86c89 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -39,7 +39,7 @@ public: size_t approximate_size() const; - Status serialize(std::string& group_id); + Status serialize(std::string& table_id); ~MemVectors(); @@ -70,15 +70,15 @@ public: MemManager(const std::shared_ptr& meta_, const Options& options) : _pMeta(meta_), options_(options) {} - MemVectorsPtr get_mem_by_group(const std::string& group_id_); + MemVectorsPtr get_mem_by_group(const std::string& table_id_); - Status add_vectors(const std::string& group_id_, + Status add_vectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); - Status serialize(std::vector& group_ids); + Status serialize(std::vector& table_ids); private: - Status add_vectors_no_lock(const std::string& group_id_, + Status add_vectors_no_lock(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_); Status mark_memory_as_immutable(); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index afa9eee2d7..90b3384c61 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -24,31 +24,31 @@ public: virtual Status add_group(TableSchema& group_info) = 0; virtual Status get_group(TableSchema& group_info) = 0; - virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; + virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; - virtual Status delete_group_partitions(const std::string& group_id, + virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status has_group_file(const std::string& group_id_, + virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, bool& has_or_not_) = 0; - virtual Status get_group_file(const std::string& group_id_, + virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0; - virtual Status get_group_files(const std::string& group_id_, + virtual Status get_group_files(const std::string& table_id_, const int date_delta_, TableFilesSchema& group_files_info_) = 0; virtual Status update_files(TableFilesSchema& files) = 0; - virtual Status files_to_search(const std::string& group_id, + virtual Status files_to_search(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) = 0; - virtual Status files_to_merge(const std::string& group_id, + virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; virtual Status size(long& result) = 0; @@ -62,7 +62,7 @@ public: virtual Status drop_all() = 0; - virtual Status count(const std::string& group_id, long& result) = 0; + virtual Status count(const std::string& table_id, long& result) = 0; static DateT GetDate(const std::time_t& t, int day_delta = 0); static DateT GetDate(); diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index 88bb112de9..b2fe783323 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -20,7 +20,7 @@ typedef std::vector DatesT; struct TableSchema { size_t id; - std::string group_id; + std::string table_id; size_t files_cnt = 0; uint16_t dimension; std::string location; @@ -37,7 +37,7 @@ struct TableFileSchema { } FILE_TYPE; size_t id; - std::string group_id; + std::string table_id; std::string file_id; int file_type = NEW; size_t size; diff --git a/cpp/src/db/db_connection.cpp b/cpp/src/db/db_connection.cpp deleted file mode 100644 index 74779daf18..0000000000 --- a/cpp/src/db/db_connection.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "db_connection.h" - - -namespace zilliz { -namespace vecwise { -namespace engine { - -using std::string; -using namespace sqlite_orm; - -string storage_file_name = "default.sqlite"; - -SqliteDBPtr connect() { - SqliteDBPtr temp = std::make_shared(initStorage(storage_file_name)); - temp->sync_schema(); - temp->open_forever(); // thread safe option - //temp->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log - return temp; -} - -/* SqliteDBPtr Connection::connect_ = connect(); */ - -} -} -} diff --git a/cpp/src/db/db_connection.h b/cpp/src/db/db_connection.h deleted file mode 100644 index cfd34461af..0000000000 --- a/cpp/src/db/db_connection.h +++ /dev/null @@ -1,64 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#pragma once - -#include -#include - -#include - - -namespace zilliz { -namespace vecwise { -namespace engine { - -struct GroupSchema { - size_t id; - std::string group_id; - size_t files_cnt = 0; - uint16_t dimension; - std::string location = ""; - std::string next_file_location = ""; -}; // GroupSchema - - -struct GroupFileSchema { - typedef enum { - RAW, - INDEX - } FILE_TYPE; - - size_t id; - std::string group_id; - std::string file_id; - int files_type = RAW; - size_t rows; - std::string location = ""; -}; // GroupFileSchema - -inline auto initStorage(const std::string &path) { - using namespace sqlite_orm; - return make_storage(path, - // Add table below - make_table("Groups", - make_column("id", &GroupSchema::id, primary_key()), - make_column("group_id", &GroupSchema::group_id, unique()), - make_column("dimension", &GroupSchema::dimension), - make_column("files_cnt", &GroupSchema::files_cnt, default_value(0)))); -} - -using SqliteDB = decltype(initStorage("")); -using SqliteDBPtr= std::shared_ptr; - -class Connection { - protected: - static SqliteDBPtr connect_; -}; - -} -} -} diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index c2d37a5e15..381fa28e95 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -68,11 +68,11 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -113,11 +113,11 @@ TEST_F(DBTest, DB_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -199,11 +199,11 @@ TEST_F(DBTest, SEARCH_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; - group_info.group_id = group_name; + group_info.table_id = group_name; engine::Status stat = db_->add_group(group_info); engine::meta::TableSchema group_info_get; - group_info_get.group_id = group_name; + group_info_get.table_id = group_name; stat = db_->get_group(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 62b1fce1b0..2f1bd4fbee 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -18,10 +18,10 @@ using namespace zilliz::vecwise::engine; TEST_F(MetaTest, GROUP_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); ASSERT_TRUE(status.ok()); @@ -30,26 +30,26 @@ TEST_F(MetaTest, GROUP_TEST) { status = impl_->get_group(group); ASSERT_TRUE(status.ok()); ASSERT_EQ(group.id, gid); - ASSERT_EQ(group.group_id, group_id); + ASSERT_EQ(group.table_id, table_id); - group.group_id = "not_found"; + group.table_id = "not_found"; status = impl_->get_group(group); ASSERT_TRUE(!status.ok()); - group.group_id = group_id; + group.table_id = table_id; status = impl_->add_group(group); ASSERT_TRUE(!status.ok()); } TEST_F(MetaTest, GROUP_FILE_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; status = impl_->add_group_file(group_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW); @@ -65,14 +65,14 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_TRUE(status.ok()); group_file.date = meta::Meta::GetDateWithDelta(-2); @@ -83,9 +83,9 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { dates.clear(); dates.push_back(group_file.date); - status = impl_->delete_group_partitions(group_file.group_id, dates); + status = impl_->delete_group_partitions(group_file.table_id, dates); ASSERT_TRUE(status.ok()); - status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); + status = impl_->get_group_file(group_file.table_id, group_file.file_id, group_file); ASSERT_TRUE(status.ok()); ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE); } @@ -100,15 +100,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { options.archive_conf = ArchiveConf("delete", ss.str()); auto impl = meta::DBMetaImpl(options); - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl.add_group(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -127,7 +127,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.group_id, file.file_id, file); + status = impl.get_group_file(file.table_id, file.file_id, file); ASSERT_TRUE(status.ok()); if (days[i] < days_num) { ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW); @@ -146,15 +146,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { options.archive_conf = ArchiveConf("delete", "disk:11"); auto impl = meta::DBMetaImpl(options); - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl.add_group(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; auto cnt = 10; auto each_size = 2UL; @@ -170,7 +170,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.group_id, file.file_id, file); + status = impl.get_group_file(file.table_id, file.file_id, file); ASSERT_TRUE(status.ok()); if (i < 5) { ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE); @@ -184,10 +184,10 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { } TEST_F(MetaTest, GROUP_FILES_TEST) { - auto group_id = "meta_test_group"; + auto table_id = "meta_test_group"; meta::TableSchema group; - group.group_id = group_id; + group.table_id = table_id; auto status = impl_->add_group(group); int new_files_cnt = 4; @@ -196,7 +196,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema group_file; - group_file.group_id = group.group_id; + group_file.table_id = group.table_id; for (auto i=0; iadd_group_file(group_file); @@ -229,7 +229,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->files_to_merge(group.group_id, dated_files); + status = impl_->files_to_merge(group.table_id, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt); @@ -238,7 +238,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatesT dates = {group_file.date}; - status = impl_->files_to_search(group_id, dates, dated_files); + status = impl_->files_to_search(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[group_file.date].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); From 3d330e0aa47a602e14c8811309858bc7840792d0 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:45:39 +0800 Subject: [PATCH 05/31] refactor(db): change group_info to table_schema and add_group to CreateTable Former-commit-id: e8d48ec8e54429bf5d1080997ca329b42ddcab82 --- cpp/src/db/DB.h | 4 +- cpp/src/db/DBImpl.cpp | 8 ++-- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/DBMetaImpl.cpp | 80 +++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/LocalMetaImpl.cpp | 60 ++++++++++++------------- cpp/src/db/LocalMetaImpl.h | 8 ++-- cpp/src/db/Meta.h | 4 +- cpp/unittest/db/meta_tests.cpp | 12 ++--- 9 files changed, 91 insertions(+), 91 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 20e4924076..1307d8ed97 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -21,8 +21,8 @@ class DB { public: static void Open(const Options& options, DB** dbptr); - virtual Status add_group(meta::TableSchema& group_info_) = 0; - virtual Status get_group(meta::TableSchema& group_info_) = 0; + virtual Status add_group(meta::TableSchema& table_schema_) = 0; + virtual Status get_group(meta::TableSchema& table_schema_) = 0; virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 54a1380031..3d202e6534 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -35,13 +35,13 @@ DBImpl::DBImpl(const Options& options) } template -Status DBImpl::add_group(meta::TableSchema& group_info) { - return _pMeta->add_group(group_info); +Status DBImpl::add_group(meta::TableSchema& table_schema) { + return _pMeta->CreateTable(table_schema); } template -Status DBImpl::get_group(meta::TableSchema& group_info) { - return _pMeta->get_group(group_info); +Status DBImpl::get_group(meta::TableSchema& table_schema) { + return _pMeta->get_group(table_schema); } template diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 35a4a93eff..19025cf2fd 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -33,8 +33,8 @@ public: DBImpl(const Options& options); - virtual Status add_group(meta::TableSchema& group_info) override; - virtual Status get_group(meta::TableSchema& group_info) override; + virtual Status add_group(meta::TableSchema& table_schema) override; + virtual Status get_group(meta::TableSchema& table_schema) override; virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 41fceb5c0f..54f65f5777 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -117,9 +117,9 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id, return Status::OK(); } - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group(table_schema); if (!status.ok()) { return status; } @@ -148,24 +148,24 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id, return Status::OK(); } -Status DBMetaImpl::add_group(TableSchema& group_info) { - if (group_info.table_id == "") { - NextGroupId(group_info.table_id); +Status DBMetaImpl::CreateTable(TableSchema& table_schema) { + if (table_schema.table_id == "") { + NextGroupId(table_schema.table_id); } - group_info.files_cnt = 0; - group_info.id = -1; - group_info.created_on = utils::GetMicroSecTimeStamp(); + table_schema.files_cnt = 0; + table_schema.id = -1; + table_schema.created_on = utils::GetMicroSecTimeStamp(); { try { - auto id = ConnectorPtr->insert(group_info); - group_info.id = id; + auto id = ConnectorPtr->insert(table_schema); + table_schema.id = id; } catch (...) { return Status::DBTransactionError("Add Group Error"); } } - auto group_path = GetGroupPath(group_info.table_id); + auto group_path = GetGroupPath(table_schema.table_id); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -178,24 +178,24 @@ Status DBMetaImpl::add_group(TableSchema& group_info) { return Status::OK(); } -Status DBMetaImpl::get_group(TableSchema& group_info) { - return get_group_no_lock(group_info); +Status DBMetaImpl::get_group(TableSchema& table_schema) { + return get_group_no_lock(table_schema); } -Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) { +Status DBMetaImpl::get_group_no_lock(TableSchema& table_schema) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id, &TableSchema::table_id, &TableSchema::files_cnt, &TableSchema::dimension), - where(c(&TableSchema::table_id) == group_info.table_id)); + where(c(&TableSchema::table_id) == table_schema.table_id)); assert(groups.size() <= 1); if (groups.size() == 1) { - group_info.id = std::get<0>(groups[0]); - group_info.files_cnt = std::get<2>(groups[0]); - group_info.dimension = std::get<3>(groups[0]); + table_schema.id = std::get<0>(groups[0]); + table_schema.files_cnt = std::get<2>(groups[0]); + table_schema.dimension = std::get<3>(groups[0]); } else { - return Status::NotFound("Group " + group_info.table_id + " not found"); + return Status::NotFound("Group " + table_schema.table_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -226,16 +226,16 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { if (group_file.date == EmptyDate) { group_file.date = Meta::GetDate(); } - TableSchema group_info; - group_info.table_id = group_file.table_id; - auto status = get_group(group_info); + TableSchema table_schema; + table_schema.table_id = group_file.table_id; + auto status = get_group(table_schema); if (!status.ok()) { return status; } NextFileId(group_file.file_id); group_file.file_type = TableFileSchema::NEW; - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; group_file.size = 0; group_file.created_on = utils::GetMicroSecTimeStamp(); group_file.updated_time = group_file.created_on; @@ -288,13 +288,13 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { GetGroupFilePath(group_file); auto groupItr = groups.find(group_file.table_id); if (groupItr == groups.end()) { - TableSchema group_info; - group_info.table_id = group_file.table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = group_file.table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } - groups[group_file.table_id] = group_info; + groups[group_file.table_id] = table_schema; } group_file.dimension = groups[group_file.table_id].dimension; files.push_back(group_file); @@ -327,9 +327,9 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX))); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -342,7 +342,7 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { @@ -372,9 +372,9 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and c(&TableFileSchema::table_id) == table_id)); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -387,7 +387,7 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, group_file.file_type = std::get<3>(file); group_file.size = std::get<4>(file); group_file.date = std::get<5>(file); - group_file.dimension = group_info.dimension; + group_file.dimension = table_schema.dimension; GetGroupFilePath(group_file); auto dateItr = files.find(group_file.date); if (dateItr == files.end()) { @@ -670,9 +670,9 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and c(&TableFileSchema::table_id) == table_id)); - TableSchema group_info; - group_info.table_id = table_id; - auto status = get_group_no_lock(group_info); + TableSchema table_schema; + table_schema.table_id = table_id; + auto status = get_group_no_lock(table_schema); if (!status.ok()) { return status; } @@ -682,7 +682,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { result += std::get<0>(file); } - result /= group_info.dimension; + result /= table_schema.dimension; } catch (std::exception & e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index b278bee268..2706e43e54 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -19,7 +19,7 @@ class DBMetaImpl : public Meta { public: DBMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(TableSchema& group_info) override; + virtual Status CreateTable(TableSchema& table_schema) override; virtual Status get_group(TableSchema& group_info_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index 70a073506d..c8c320fca0 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) { return GetGroupMetaPathByGroupPath(GetGroupPath(table_id)); } -Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) { +Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) { boost::property_tree::ptree ptree; boost::property_tree::read_json(path, ptree); auto files_cnt = ptree.get_child("files_cnt").data(); @@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem /* std::cout << dimension << std::endl; */ /* std::cout << files_cnt << std::endl; */ - group_info.id = std::stoi(group_info.table_id); - group_info.files_cnt = std::stoi(files_cnt); - group_info.dimension = std::stoi(dimension); - group_info.location = GetGroupPath(group_info.table_id); + table_schema.id = std::stoi(table_schema.table_id); + table_schema.files_cnt = std::stoi(files_cnt); + table_schema.dimension = std::stoi(dimension); + table_schema.location = GetGroupPath(table_schema.table_id); return Status::OK(); } -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) { - group_info.table_id = table_id; - return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info); +Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) { + table_schema.table_id = table_id; + return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema); } LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_) @@ -90,15 +90,15 @@ Status LocalMetaImpl::initialize() { return Status::OK(); } -Status LocalMetaImpl::add_group(TableSchema& group_info) { +Status LocalMetaImpl::CreateTable(TableSchema& table_schema) { std::string real_gid; size_t id = SimpleIDGenerator().getNextIDNumber(); - if (group_info.table_id == "") { + if (table_schema.table_id == "") { std::stringstream ss; ss << id; real_gid = ss.str(); } else { - real_gid = group_info.table_id; + real_gid = table_schema.table_id; } bool group_exist; @@ -110,27 +110,27 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) { return Status::GroupError("Cannot Create Group " + real_gid); } - group_info.table_id = real_gid; - group_info.files_cnt = 0; - group_info.id = 0; - group_info.location = GetGroupPath(real_gid); + table_schema.table_id = real_gid; + table_schema.files_cnt = 0; + table_schema.id = 0; + table_schema.location = GetGroupPath(real_gid); boost::property_tree::ptree out; - out.put("files_cnt", group_info.files_cnt); - out.put("dimension", group_info.dimension); + out.put("files_cnt", table_schema.files_cnt); + out.put("dimension", table_schema.dimension); boost::property_tree::write_json(GetGroupMetaPath(real_gid), out); return Status::OK(); } -Status LocalMetaImpl::get_group(TableSchema& group_info) { +Status LocalMetaImpl::get_group(TableSchema& table_schema) { bool group_exist; - has_group(group_info.table_id, group_exist); + has_group(table_schema.table_id, group_exist); if (!group_exist) { - return Status::NotFound("Group " + group_info.table_id + " Not Found"); + return Status::NotFound("Group " + table_schema.table_id + " Not Found"); } - return GetGroupMetaInfo(group_info.table_id, group_info); + return GetGroupMetaInfo(table_schema.table_id, table_schema); } Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { @@ -139,14 +139,14 @@ Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { } Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { - TableSchema group_info; - /* auto status = get_group(group_info); */ + TableSchema table_schema; + /* auto status = get_group(table_schema); */ /* if (!status.ok()) { */ /* return status; */ /* } */ /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */ /* group_file_info.table_id = table_id; */ - /* group_file_info.dimension = group_info.dimension; */ + /* group_file_info.dimension = table_schema.dimension; */ /* group_file_info.location = location; */ /* group_file_info.date = date; */ return Status::OK(); @@ -159,8 +159,8 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { boost::filesystem::directory_iterator end_itr; for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { auto group_path = itr->path().string(); - TableSchema group_info; - GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); + TableSchema table_schema; + GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { auto partition_path = innerItr->path().string(); for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { @@ -172,7 +172,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { TableFileSchema f; f.location = location; /* f.table_id = table_id; */ - f.dimension = group_info.dimension; + f.dimension = table_schema.dimension; files.push_back(f); } } @@ -188,8 +188,8 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id, /* boost::filesystem::directory_iterator end_itr; */ /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ /* auto group_path = itr->path().string(); */ - /* TableSchema group_info; */ - /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */ + /* TableSchema table_schema; */ + /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */ /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ /* auto partition_path = innerItr->path().string(); */ /* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */ @@ -201,7 +201,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id, /* TableFileSchema f; */ /* f.location = location; */ /* f.table_id = table_id; */ - /* f.dimension = group_info.dimension; */ + /* f.dimension = table_schema.dimension; */ /* files.push_back(f); */ /* } */ /* } */ diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index e57f0a97c1..46ecdd4145 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -18,8 +18,8 @@ public: const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; LocalMetaImpl(const DBMetaOptions& options_); - virtual Status add_group(TableSchema& group_info_) override; - virtual Status get_group(TableSchema& group_info_) override; + virtual Status CreateTable(TableSchema& table_schema) override; + virtual Status get_group(TableSchema& table_schema_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; @@ -59,9 +59,9 @@ public: private: - Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info); + Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema); std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info); + Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema); std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, TableFileSchema::FILE_TYPE file_type); std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 90b3384c61..014b04c917 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -22,8 +22,8 @@ class Meta { public: using Ptr = std::shared_ptr; - virtual Status add_group(TableSchema& group_info) = 0; - virtual Status get_group(TableSchema& group_info) = 0; + virtual Status CreateTable(TableSchema& table_schema) = 0; + virtual Status get_group(TableSchema& table_schema) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 2f1bd4fbee..0c8d52f783 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -22,7 +22,7 @@ TEST_F(MetaTest, GROUP_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); ASSERT_TRUE(status.ok()); auto gid = group.id; @@ -37,7 +37,7 @@ TEST_F(MetaTest, GROUP_TEST) { ASSERT_TRUE(!status.ok()); group.table_id = table_id; - status = impl_->add_group(group); + status = impl_->CreateTable(group); ASSERT_TRUE(!status.ok()); } @@ -46,7 +46,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); meta::TableFileSchema group_file; group_file.table_id = group.table_id; @@ -104,7 +104,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { meta::TableSchema group; group.table_id = table_id; - auto status = impl.add_group(group); + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; @@ -150,7 +150,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { meta::TableSchema group; group.table_id = table_id; - auto status = impl.add_group(group); + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema group_file; @@ -188,7 +188,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { meta::TableSchema group; group.table_id = table_id; - auto status = impl_->add_group(group); + auto status = impl_->CreateTable(group); int new_files_cnt = 4; int raw_files_cnt = 5; From 624e34f10d4de80e5827e67a632c8d3f47476fb1 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 17:54:00 +0800 Subject: [PATCH 06/31] refactor(db): refactor DescribeTable Former-commit-id: 021d731f3be6f338423caabef59bdd4c46ed02d8 --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 18 +++++++----------- cpp/src/db/DBMetaImpl.h | 3 +-- cpp/src/db/LocalMetaImpl.cpp | 10 +++++----- cpp/src/db/LocalMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 4 ++-- 7 files changed, 18 insertions(+), 23 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 3d202e6534..3a92f41d3e 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -41,7 +41,7 @@ Status DBImpl::add_group(meta::TableSchema& table_schema) { template Status DBImpl::get_group(meta::TableSchema& table_schema) { - return _pMeta->get_group(table_schema); + return _pMeta->DescribeTable(table_schema); } template diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 54f65f5777..302efb4ec1 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -119,7 +119,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id, TableSchema table_schema; table_schema.table_id = table_id; - auto status = get_group(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } @@ -178,11 +178,7 @@ Status DBMetaImpl::CreateTable(TableSchema& table_schema) { return Status::OK(); } -Status DBMetaImpl::get_group(TableSchema& table_schema) { - return get_group_no_lock(table_schema); -} - -Status DBMetaImpl::get_group_no_lock(TableSchema& table_schema) { +Status DBMetaImpl::DescribeTable(TableSchema& table_schema) { try { auto groups = ConnectorPtr->select(columns(&TableSchema::id, &TableSchema::table_id, @@ -228,7 +224,7 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { } TableSchema table_schema; table_schema.table_id = group_file.table_id; - auto status = get_group(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } @@ -290,7 +286,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { if (groupItr == groups.end()) { TableSchema table_schema; table_schema.table_id = group_file.table_id; - auto status = get_group_no_lock(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } @@ -329,7 +325,7 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, TableSchema table_schema; table_schema.table_id = table_id; - auto status = get_group_no_lock(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } @@ -374,7 +370,7 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, TableSchema table_schema; table_schema.table_id = table_id; - auto status = get_group_no_lock(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } @@ -672,7 +668,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { TableSchema table_schema; table_schema.table_id = table_id; - auto status = get_group_no_lock(table_schema); + auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 2706e43e54..98ee19a8f6 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -20,7 +20,7 @@ public: DBMetaImpl(const DBMetaOptions& options_); virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status get_group(TableSchema& group_info_) override; + virtual Status DescribeTable(TableSchema& group_info_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; @@ -68,7 +68,6 @@ private: Status NextFileId(std::string& file_id); Status NextGroupId(std::string& table_id); Status discard_files_of_size(long to_discard_size); - Status get_group_no_lock(TableSchema& group_info); std::string GetGroupPath(const std::string& table_id); std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); void GetGroupFilePath(TableFileSchema& group_file); diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp index c8c320fca0..70d30c2a5c 100644 --- a/cpp/src/db/LocalMetaImpl.cpp +++ b/cpp/src/db/LocalMetaImpl.cpp @@ -123,11 +123,11 @@ Status LocalMetaImpl::CreateTable(TableSchema& table_schema) { return Status::OK(); } -Status LocalMetaImpl::get_group(TableSchema& table_schema) { - bool group_exist; - has_group(table_schema.table_id, group_exist); - if (!group_exist) { - return Status::NotFound("Group " + table_schema.table_id + " Not Found"); +Status LocalMetaImpl::DescribeTable(TableSchema& table_schema) { + bool exist; + has_group(table_schema.table_id, exist); + if (!exist) { + return Status::NotFound("Table " + table_schema.table_id + " Not Found"); } return GetGroupMetaInfo(table_schema.table_id, table_schema); diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h index 46ecdd4145..8f55965543 100644 --- a/cpp/src/db/LocalMetaImpl.h +++ b/cpp/src/db/LocalMetaImpl.h @@ -19,7 +19,7 @@ public: LocalMetaImpl(const DBMetaOptions& options_); virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status get_group(TableSchema& table_schema_) override; + virtual Status DescribeTable(TableSchema& table_schema_) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 014b04c917..20d510c153 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -23,7 +23,7 @@ public: using Ptr = std::shared_ptr; virtual Status CreateTable(TableSchema& table_schema) = 0; - virtual Status get_group(TableSchema& table_schema) = 0; + virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 0c8d52f783..b74a65c9cf 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -27,13 +27,13 @@ TEST_F(MetaTest, GROUP_TEST) { auto gid = group.id; group.id = -1; - status = impl_->get_group(group); + status = impl_->DescribeTable(group); ASSERT_TRUE(status.ok()); ASSERT_EQ(group.id, gid); ASSERT_EQ(group.table_id, table_id); group.table_id = "not_found"; - status = impl_->get_group(group); + status = impl_->DescribeTable(group); ASSERT_TRUE(!status.ok()); group.table_id = table_id; From 9d4591612d4e3178414158d4dacb198be917d840 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:02:22 +0800 Subject: [PATCH 07/31] refactor(db): delete local meta impl and refactor HasTable Former-commit-id: 4849b994d40cf433797892eb37ceeac46fc65d3d --- cpp/src/db/DBImpl.cpp | 4 +- cpp/src/db/DBMetaImpl.cpp | 8 +- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/LocalMetaImpl.cpp | 277 ----------------------------------- cpp/src/db/LocalMetaImpl.h | 83 ----------- cpp/src/db/Meta.h | 2 +- 6 files changed, 8 insertions(+), 368 deletions(-) delete mode 100644 cpp/src/db/LocalMetaImpl.cpp delete mode 100644 cpp/src/db/LocalMetaImpl.h diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 3a92f41d3e..4d2d51fe33 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -51,8 +51,8 @@ Status DBImpl::delete_vectors(const std::string& table_id, } template -Status DBImpl::has_group(const std::string& table_id_, bool& has_or_not_) { - return _pMeta->has_group(table_id_, has_or_not_); +Status DBImpl::has_group(const std::string& table_id, bool& has_or_not) { + return _pMeta->HasTable(table_id, has_or_not); } template diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 302efb4ec1..ec2548c137 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -201,12 +201,12 @@ Status DBMetaImpl::DescribeTable(TableSchema& table_schema) { return Status::OK(); } -Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { +Status DBMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { try { - auto groups = ConnectorPtr->select(columns(&TableSchema::id), + auto tables = ConnectorPtr->select(columns(&TableSchema::id), where(c(&TableSchema::table_id) == table_id)); - assert(groups.size() <= 1); - if (groups.size() == 1) { + assert(tables.size() <= 1); + if (tables.size() == 1) { has_or_not = true; } else { has_or_not = false; diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 98ee19a8f6..014cc3d693 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -21,7 +21,7 @@ public: virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; + virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; virtual Status add_group_file(TableFileSchema& group_file_info) override; virtual Status delete_group_partitions(const std::string& table_id, diff --git a/cpp/src/db/LocalMetaImpl.cpp b/cpp/src/db/LocalMetaImpl.cpp deleted file mode 100644 index 70d30c2a5c..0000000000 --- a/cpp/src/db/LocalMetaImpl.cpp +++ /dev/null @@ -1,277 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include -#include -#include - -#include -#include "LocalMetaImpl.h" -#include "IDGenerator.h" - -namespace zilliz { -namespace vecwise { -namespace engine { -namespace meta { - -long LocalMetaImpl::GetFileSize(const std::string& filename) -{ - struct stat stat_buf; - int rc = stat(filename.c_str(), &stat_buf); - return rc == 0 ? stat_buf.st_size : -1; -} - -std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) { - return _options.path + "/" + table_id; -} - -std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { - std::stringstream ss; - ss << GetGroupPath(table_id) << "/" << date; - return ss.str(); -} - -std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, - TableFileSchema::FILE_TYPE file_type) { - std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index"; - SimpleIDGenerator g; - std::stringstream ss; - ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; - return ss.str(); -} - -std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_path) { - return group_path + "/" + "meta"; -} - -std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) { - return GetGroupMetaPathByGroupPath(GetGroupPath(table_id)); -} - -Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) { - boost::property_tree::ptree ptree; - boost::property_tree::read_json(path, ptree); - auto files_cnt = ptree.get_child("files_cnt").data(); - auto dimension = ptree.get_child("dimension").data(); - /* std::cout << dimension << std::endl; */ - /* std::cout << files_cnt << std::endl; */ - - table_schema.id = std::stoi(table_schema.table_id); - table_schema.files_cnt = std::stoi(files_cnt); - table_schema.dimension = std::stoi(dimension); - table_schema.location = GetGroupPath(table_schema.table_id); - - return Status::OK(); - -} - -Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) { - table_schema.table_id = table_id; - return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema); -} - -LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_) - : _options(options_) { - initialize(); -} - -Status LocalMetaImpl::initialize() { - if (boost::filesystem::is_directory(_options.path)) { - } - else if (!boost::filesystem::create_directory(_options.path)) { - return Status::InvalidDBPath("Cannot Create " + _options.path); - } - return Status::OK(); -} - -Status LocalMetaImpl::CreateTable(TableSchema& table_schema) { - std::string real_gid; - size_t id = SimpleIDGenerator().getNextIDNumber(); - if (table_schema.table_id == "") { - std::stringstream ss; - ss << id; - real_gid = ss.str(); - } else { - real_gid = table_schema.table_id; - } - - bool group_exist; - has_group(real_gid, group_exist); - if (group_exist) { - return Status::GroupError("Group Already Existed " + real_gid); - } - if (!boost::filesystem::create_directory(GetGroupPath(real_gid))) { - return Status::GroupError("Cannot Create Group " + real_gid); - } - - table_schema.table_id = real_gid; - table_schema.files_cnt = 0; - table_schema.id = 0; - table_schema.location = GetGroupPath(real_gid); - - boost::property_tree::ptree out; - out.put("files_cnt", table_schema.files_cnt); - out.put("dimension", table_schema.dimension); - boost::property_tree::write_json(GetGroupMetaPath(real_gid), out); - - return Status::OK(); -} - -Status LocalMetaImpl::DescribeTable(TableSchema& table_schema) { - bool exist; - has_group(table_schema.table_id, exist); - if (!exist) { - return Status::NotFound("Table " + table_schema.table_id + " Not Found"); - } - - return GetGroupMetaInfo(table_schema.table_id, table_schema); -} - -Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) { - has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id)); - return Status::OK(); -} - -Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) { - TableSchema table_schema; - /* auto status = get_group(table_schema); */ - /* if (!status.ok()) { */ - /* return status; */ - /* } */ - /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */ - /* group_file_info.table_id = table_id; */ - /* group_file_info.dimension = table_schema.dimension; */ - /* group_file_info.location = location; */ - /* group_file_info.date = date; */ - return Status::OK(); -} - -Status LocalMetaImpl::files_to_index(TableFilesSchema& files) { - files.clear(); - - std::string suffix; - boost::filesystem::directory_iterator end_itr; - for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { - auto group_path = itr->path().string(); - TableSchema table_schema; - GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); - for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { - auto partition_path = innerItr->path().string(); - for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { - auto location = fItr->path().string(); - suffix = location.substr(location.find_last_of('.') + 1); - if (suffix == "index") continue; - if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue; - std::cout << "[About to index] " << location << std::endl; - TableFileSchema f; - f.location = location; - /* f.table_id = table_id; */ - f.dimension = table_schema.dimension; - files.push_back(f); - } - } - } - - return Status::OK(); -} - -Status LocalMetaImpl::files_to_merge(const std::string& table_id, - DatePartionedTableFilesSchema& files) { - files.clear(); - /* std::string suffix; */ - /* boost::filesystem::directory_iterator end_itr; */ - /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ - /* auto group_path = itr->path().string(); */ - /* TableSchema table_schema; */ - /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */ - /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ - /* auto partition_path = innerItr->path().string(); */ - /* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */ - /* auto location = fItr->path().string(); */ - /* suffix = location.substr(location.find_last_of('.') + 1); */ - /* if (suffix == "index") continue; */ - /* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */ - /* std::cout << "[About to index] " << location << std::endl; */ - /* TableFileSchema f; */ - /* f.location = location; */ - /* f.table_id = table_id; */ - /* f.dimension = table_schema.dimension; */ - /* files.push_back(f); */ - /* } */ - /* } */ - /* } */ - - return Status::OK(); -} - -Status LocalMetaImpl::has_group_file(const std::string& table_id_, - const std::string& file_id_, - bool& has_or_not_) { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::get_group_file(const std::string& table_id_, - const std::string& file_id_, - TableFileSchema& group_file_info_) { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::get_group_files(const std::string& table_id_, - const int date_delta_, - TableFilesSchema& group_files_info_) { - // PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::update_files(TableFilesSchema& files) { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::archive_files() { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::cleanup() { - //PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) { - // PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::drop_all() { - // PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::size(long& result) { - // PXU TODO - return Status::OK(); -} - -Status LocalMetaImpl::count(const std::string& table_id, long& result) { - // PXU TODO - return Status::OK(); -} - -} // namespace meta -} // namespace engine -} // namespace vecwise -} // namespace zilliz diff --git a/cpp/src/db/LocalMetaImpl.h b/cpp/src/db/LocalMetaImpl.h deleted file mode 100644 index 8f55965543..0000000000 --- a/cpp/src/db/LocalMetaImpl.h +++ /dev/null @@ -1,83 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#pragma once - -#include "Meta.h" -#include "Options.h" - -namespace zilliz { -namespace vecwise { -namespace engine { -namespace meta { - -class LocalMetaImpl : public Meta { -public: - const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; - LocalMetaImpl(const DBMetaOptions& options_); - - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& table_schema_) override; - virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; - - virtual Status add_group_file(TableFileSchema& group_file_info) override; - /* virtual Status delete_group_partitions(const std::string& table_id, */ - /* const meta::DatesT& dates) override; */ - - virtual Status has_group_file(const std::string& table_id_, - const std::string& file_id_, - bool& has_or_not_) override; - virtual Status get_group_file(const std::string& table_id_, - const std::string& file_id_, - TableFileSchema& group_file_info_) override; - virtual Status update_group_file(TableFileSchema& group_file_) override; - - virtual Status get_group_files(const std::string& table_id_, - const int date_delta_, - TableFilesSchema& group_files_info_) override; - - virtual Status update_files(TableFilesSchema& files) override; - - virtual Status cleanup() override; - - virtual Status files_to_merge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; - - virtual Status files_to_index(TableFilesSchema&) override; - - virtual Status archive_files() override; - - virtual Status cleanup_ttl_files(uint16_t seconds) override; - - virtual Status count(const std::string& table_id, long& result) override; - - virtual Status drop_all() override; - - virtual Status size(long& result) override; - -private: - - Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema); - std::string GetGroupMetaPathByGroupPath(const std::string& group_path); - Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema); - std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date, - TableFileSchema::FILE_TYPE file_type); - std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); - std::string GetGroupPath(const std::string& table_id); - std::string GetGroupMetaPath(const std::string& table_id); - - Status CreateGroupMeta(const TableSchema& group_schema); - long GetFileSize(const std::string& filename); - - Status initialize(); - - const DBMetaOptions _options; - -}; // LocalMetaImpl - -} // namespace meta -} // namespace engine -} // namespace vecwise -} // namespace zilliz diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 20d510c153..9ad78a5b02 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -24,7 +24,7 @@ public: virtual Status CreateTable(TableSchema& table_schema) = 0; virtual Status DescribeTable(TableSchema& table_schema) = 0; - virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; + virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0; virtual Status delete_group_partitions(const std::string& table_id, From 01e4e605e2c29558354454f8a6b97c4819aefe0a Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:12:56 +0800 Subject: [PATCH 08/31] refactor(db): CreateTableFile Former-commit-id: c29b8ccc97cc327ec9c278f2ea5456369365c4f3 --- cpp/src/db/DBImpl.cpp | 38 ++++++------ cpp/src/db/DBMetaImpl.cpp | 28 ++++----- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/MemManager.cpp | 8 +-- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 102 ++++++++++++++++----------------- 6 files changed, 90 insertions(+), 90 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4d2d51fe33..0a09386e2a 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -259,17 +259,17 @@ void DBImpl::background_call() { template Status DBImpl::merge_files(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { - meta::TableFileSchema group_file; - group_file.table_id = table_id; - group_file.date = date; - Status status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = table_id; + table_file.date = date; + Status status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { LOG(INFO) << status.ToString() << std::endl; return status; } - EngineT index(group_file.dimension, group_file.location); + EngineT index(table_file.dimension, table_file.location); meta::TableFilesSchema updated; long index_size = 0; @@ -288,14 +288,14 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat index.Serialize(); if (index_size >= _options.index_trigger_size) { - group_file.file_type = meta::TableFileSchema::TO_INDEX; + table_file.file_type = meta::TableFileSchema::TO_INDEX; } else { - group_file.file_type = meta::TableFileSchema::RAW; + table_file.file_type = meta::TableFileSchema::RAW; } - group_file.size = index_size; - updated.push_back(group_file); + table_file.size = index_size; + updated.push_back(table_file); status = _pMeta->update_files(updated); - LOG(DEBUG) << "New merged file " << group_file.file_id << + LOG(DEBUG) << "New merged file " << table_file.file_id << " of size=" << index.PhysicalSize()/(1024*1024) << " M"; index.Cache(); @@ -337,10 +337,10 @@ Status DBImpl::background_merge_files(const std::string& table_id) { template Status DBImpl::build_index(const meta::TableFileSchema& file) { - meta::TableFileSchema group_file; - group_file.table_id = file.table_id; - group_file.date = file.date; - Status status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = file.table_id; + table_file.date = file.date; + Status status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { return status; } @@ -348,18 +348,18 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { EngineT to_index(file.dimension, file.location); to_index.Load(); - auto index = to_index.BuildIndex(group_file.location); + auto index = to_index.BuildIndex(table_file.location); - group_file.file_type = meta::TableFileSchema::INDEX; - group_file.size = index->Size(); + table_file.file_type = meta::TableFileSchema::INDEX; + table_file.size = index->Size(); auto to_remove = file; to_remove.file_type = meta::TableFileSchema::TO_DELETE; - meta::TableFilesSchema update_files = {to_remove, group_file}; + meta::TableFilesSchema update_files = {to_remove, table_file}; _pMeta->update_files(update_files); - LOG(DEBUG) << "New index file " << group_file.file_id << " of size " + LOG(DEBUG) << "New index file " << table_file.file_id << " of size " << index->PhysicalSize()/(1024*1024) << " M" << " from file " << to_remove.file_id; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index ec2548c137..b04445cc70 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -218,35 +218,35 @@ Status DBMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { return Status::OK(); } -Status DBMetaImpl::add_group_file(TableFileSchema& group_file) { - if (group_file.date == EmptyDate) { - group_file.date = Meta::GetDate(); +Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) { + if (file_schema.date == EmptyDate) { + file_schema.date = Meta::GetDate(); } TableSchema table_schema; - table_schema.table_id = group_file.table_id; + table_schema.table_id = file_schema.table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - NextFileId(group_file.file_id); - group_file.file_type = TableFileSchema::NEW; - group_file.dimension = table_schema.dimension; - group_file.size = 0; - group_file.created_on = utils::GetMicroSecTimeStamp(); - group_file.updated_time = group_file.created_on; - GetGroupFilePath(group_file); + NextFileId(file_schema.file_id); + file_schema.file_type = TableFileSchema::NEW; + file_schema.dimension = table_schema.dimension; + file_schema.size = 0; + file_schema.created_on = utils::GetMicroSecTimeStamp(); + file_schema.updated_time = file_schema.created_on; + GetGroupFilePath(file_schema); { try { - auto id = ConnectorPtr->insert(group_file); - group_file.id = id; + auto id = ConnectorPtr->insert(file_schema); + file_schema.id = id; } catch (...) { return Status::DBTransactionError("Add file Error"); } } - auto partition_path = GetGroupDatePartitionPath(group_file.table_id, group_file.date); + auto partition_path = GetGroupDatePartitionPath(file_schema.table_id, file_schema.date); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 014cc3d693..f1350cceda 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -23,7 +23,7 @@ public: virtual Status DescribeTable(TableSchema& group_info_) override; virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status add_group_file(TableFileSchema& group_file_info) override; + virtual Status CreateTableFile(TableFileSchema& file_schema) override; virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) override; diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index bd69bbc2a8..60553c7898 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -85,14 +85,14 @@ typename MemManager::MemVectorsPtr MemManager::get_mem_by_grou return memIt->second; } - meta::TableFileSchema group_file; - group_file.table_id = table_id; - auto status = _pMeta->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = table_id; + auto status = _pMeta->CreateTableFile(table_file); if (!status.ok()) { return nullptr; } - _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, group_file, options_)); + _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, table_file, options_)); return _memMap[table_id]; } diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 9ad78a5b02..6fd00a9663 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -26,7 +26,7 @@ public: virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; - virtual Status add_group_file(TableFileSchema& group_file_info) = 0; + virtual Status CreateTableFile(TableFileSchema& file_schema) = 0; virtual Status delete_group_partitions(const std::string& table_id, const meta::DatesT& dates) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index b74a65c9cf..0c8b3f34b9 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -41,53 +41,53 @@ TEST_F(MetaTest, GROUP_TEST) { ASSERT_TRUE(!status.ok()); } -TEST_F(MetaTest, GROUP_FILE_TEST) { +TEST_F(MetaTest, table_file_TEST) { auto table_id = "meta_test_group"; meta::TableSchema group; group.table_id = table_id; auto status = impl_->CreateTable(group); - meta::TableFileSchema group_file; - group_file.table_id = group.table_id; - status = impl_->add_group_file(group_file); + meta::TableFileSchema table_file; + table_file.table_id = group.table_id; + status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type, meta::TableFileSchema::NEW); - auto file_id = group_file.file_id; + auto file_id = table_file.file_id; auto new_file_type = meta::TableFileSchema::INDEX; - group_file.file_type = new_file_type; + table_file.file_type = new_file_type; - status = impl_->update_group_file(group_file); + status = impl_->update_group_file(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.file_type, new_file_type); + ASSERT_EQ(table_file.file_type, new_file_type); meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->delete_group_partitions(group_file.table_id, dates); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->delete_group_partitions(group_file.table_id, dates); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_TRUE(status.ok()); - group_file.date = meta::Meta::GetDateWithDelta(-2); - status = impl_->update_group_file(group_file); + table_file.date = meta::Meta::GetDateWithDelta(-2); + status = impl_->update_group_file(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2)); - ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE); dates.clear(); - dates.push_back(group_file.date); - status = impl_->delete_group_partitions(group_file.table_id, dates); + dates.push_back(table_file.date); + status = impl_->delete_group_partitions(table_file.table_id, dates); ASSERT_TRUE(status.ok()); - status = impl_->get_group_file(group_file.table_id, group_file.file_id, group_file); + status = impl_->get_group_file(table_file.table_id, table_file.file_id, table_file); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE); + ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -107,19 +107,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { auto status = impl.CreateTable(group); meta::TableFilesSchema files; - meta::TableFileSchema group_file; - group_file.table_id = group.table_id; + meta::TableFileSchema table_file; + table_file.table_id = group.table_id; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); std::vector days; for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::NEW; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::NEW; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::RAW; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::RAW; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::TO_INDEX; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::TO_INDEX; + status = impl_->update_group_file(table_file); } for (auto i=0; iadd_group_file(group_file); - group_file.file_type = meta::TableFileSchema::INDEX; - status = impl_->update_group_file(group_file); + status = impl_->CreateTableFile(table_file); + table_file.file_type = meta::TableFileSchema::INDEX; + status = impl_->update_group_file(table_file); } meta::TableFilesSchema files; @@ -231,15 +231,15 @@ TEST_F(MetaTest, GROUP_FILES_TEST) { meta::DatePartionedTableFilesSchema dated_files; status = impl_->files_to_merge(group.table_id, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[group_file.date].size(), raw_files_cnt); + ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt); status = impl_->files_to_index(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatesT dates = {group_file.date}; + meta::DatesT dates = {table_file.date}; status = impl_->files_to_search(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); - ASSERT_EQ(dated_files[group_file.date].size(), + ASSERT_EQ(dated_files[table_file.date].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); } From fbe1278a3ab0a68e551abfb73d8f1dd779b50f77 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:17:13 +0800 Subject: [PATCH 09/31] refactor(db): DropPartitionsByDates Former-commit-id: 2534fa34f690ba60788e3619b6a87a9a7ed8650e --- cpp/src/db/DB.h | 2 -- cpp/src/db/DBImpl.cpp | 6 ------ cpp/src/db/DBImpl.h | 1 - cpp/src/db/DBMetaImpl.cpp | 4 ++-- cpp/src/db/DBMetaImpl.h | 4 ++-- cpp/src/db/Meta.h | 4 ++-- cpp/unittest/db/meta_tests.cpp | 6 +++--- 7 files changed, 9 insertions(+), 18 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 1307d8ed97..b120927ef5 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -23,8 +23,6 @@ public: virtual Status add_group(meta::TableSchema& table_schema_) = 0; virtual Status get_group(meta::TableSchema& table_schema_) = 0; - virtual Status delete_vectors(const std::string& table_id, - const meta::DatesT& dates) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; virtual Status get_group_files(const std::string& table_id_, const int date_delta_, diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 0a09386e2a..2cbf6900c9 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -44,12 +44,6 @@ Status DBImpl::get_group(meta::TableSchema& table_schema) { return _pMeta->DescribeTable(table_schema); } -template -Status DBImpl::delete_vectors(const std::string& table_id, - const meta::DatesT& dates) { - return _pMeta->delete_group_partitions(table_id, dates); -} - template Status DBImpl::has_group(const std::string& table_id, bool& has_or_not) { return _pMeta->HasTable(table_id, has_or_not); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 19025cf2fd..55741a5937 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -35,7 +35,6 @@ public: virtual Status add_group(meta::TableSchema& table_schema) override; virtual Status get_group(meta::TableSchema& table_schema) override; - virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; virtual Status get_group_files(const std::string& table_id_, diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index b04445cc70..1686a45233 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -111,8 +111,8 @@ Status DBMetaImpl::initialize() { } // PXU TODO: Temp solution. Will fix later -Status DBMetaImpl::delete_group_partitions(const std::string& table_id, - const meta::DatesT& dates) { +Status DBMetaImpl::DropPartitionsByDates(const std::string& table_id, + const DatesT& dates) { if (dates.size() == 0) { return Status::OK(); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index f1350cceda..399c891474 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -24,8 +24,8 @@ public: virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status delete_group_partitions(const std::string& table_id, - const meta::DatesT& dates) override; + virtual Status DropPartitionsByDates(const std::string& table_id, + const DatesT& dates) override; virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 6fd00a9663..052ff4956b 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -27,8 +27,8 @@ public: virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; virtual Status CreateTableFile(TableFileSchema& file_schema) = 0; - virtual Status delete_group_partitions(const std::string& table_id, - const meta::DatesT& dates) = 0; + virtual Status DropPartitionsByDates(const std::string& table_id, + const DatesT& dates) = 0; virtual Status has_group_file(const std::string& table_id_, const std::string& file_id_, diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 0c8b3f34b9..b016b947e4 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -65,14 +65,14 @@ TEST_F(MetaTest, table_file_TEST) { meta::DatesT dates; dates.push_back(meta::Meta::GetDate()); - status = impl_->delete_group_partitions(table_file.table_id, dates); + status = impl_->DropPartitionsByDates(table_file.table_id, dates); ASSERT_FALSE(status.ok()); dates.clear(); for (auto i=2; i < 10; ++i) { dates.push_back(meta::Meta::GetDateWithDelta(-1*i)); } - status = impl_->delete_group_partitions(table_file.table_id, dates); + status = impl_->DropPartitionsByDates(table_file.table_id, dates); ASSERT_TRUE(status.ok()); table_file.date = meta::Meta::GetDateWithDelta(-2); @@ -83,7 +83,7 @@ TEST_F(MetaTest, table_file_TEST) { dates.clear(); dates.push_back(table_file.date); - status = impl_->delete_group_partitions(table_file.table_id, dates); + status = impl_->DropPartitionsByDates(table_file.table_id, dates); ASSERT_TRUE(status.ok()); status = impl_->get_group_file(table_file.table_id, table_file.file_id, table_file); ASSERT_TRUE(status.ok()); From 59fa814af4601b627b1234a0588e6dbb428ebcca Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:20:11 +0800 Subject: [PATCH 10/31] refactor(db): delete has_group_file Former-commit-id: 821b2784c727d30375b7ce5c2bf3e9e9aef16fd8 --- cpp/src/db/DBMetaImpl.cpp | 7 ------- cpp/src/db/DBMetaImpl.h | 3 --- cpp/src/db/Meta.h | 3 --- 3 files changed, 13 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 1686a45233..d49e20a147 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -399,13 +399,6 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, return Status::OK(); } -Status DBMetaImpl::has_group_file(const std::string& table_id_, - const std::string& file_id_, - bool& has_or_not_) { - //PXU TODO - return Status::OK(); -} - Status DBMetaImpl::get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) { diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 399c891474..595e3e7f2f 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -27,9 +27,6 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - virtual Status has_group_file(const std::string& table_id_, - const std::string& file_id_, - bool& has_or_not_) override; virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 052ff4956b..3923df7fb1 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -30,9 +30,6 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; - virtual Status has_group_file(const std::string& table_id_, - const std::string& file_id_, - bool& has_or_not_) = 0; virtual Status get_group_file(const std::string& table_id_, const std::string& file_id_, TableFileSchema& group_file_info_) = 0; From ba8929e9a467748dcb076ddb25d1a0621c79dc44 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:30:58 +0800 Subject: [PATCH 11/31] refactor(db): GetTableFile Former-commit-id: 9b5f8d3a6c3202acc5b16f9b75f4ebe9cb2c6c50 --- cpp/src/db/DBMetaImpl.cpp | 24 ++++++++++++------------ cpp/src/db/DBMetaImpl.h | 5 ++--- cpp/src/db/Meta.h | 4 +--- cpp/unittest/db/meta_tests.cpp | 6 +++--- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index d49e20a147..f80e1281e1 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -399,9 +399,8 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, return Status::OK(); } -Status DBMetaImpl::get_group_file(const std::string& table_id_, - const std::string& file_id_, - TableFileSchema& group_file_info_) { +Status DBMetaImpl::GetTableFile(TableFileSchema& file_schema) { + try { auto files = ConnectorPtr->select(columns(&TableFileSchema::id, &TableFileSchema::table_id, @@ -409,19 +408,20 @@ Status DBMetaImpl::get_group_file(const std::string& table_id_, &TableFileSchema::file_type, &TableFileSchema::size, &TableFileSchema::date), - where(c(&TableFileSchema::file_id) == file_id_ and - c(&TableFileSchema::table_id) == table_id_ + where(c(&TableFileSchema::file_id) == file_schema.file_id and + c(&TableFileSchema::table_id) == file_schema.table_id )); assert(files.size() <= 1); if (files.size() == 1) { - group_file_info_.id = std::get<0>(files[0]); - group_file_info_.table_id = std::get<1>(files[0]); - group_file_info_.file_id = std::get<2>(files[0]); - group_file_info_.file_type = std::get<3>(files[0]); - group_file_info_.size = std::get<4>(files[0]); - group_file_info_.date = std::get<5>(files[0]); + file_schema.id = std::get<0>(files[0]); + file_schema.table_id = std::get<1>(files[0]); + file_schema.file_id = std::get<2>(files[0]); + file_schema.file_type = std::get<3>(files[0]); + file_schema.size = std::get<4>(files[0]); + file_schema.date = std::get<5>(files[0]); } else { - return Status::NotFound("GroupFile " + file_id_ + " not found"); + return Status::NotFound("Table:" + file_schema.table_id + + " File:" + file_schema.file_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 595e3e7f2f..e2b5a5028e 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -27,9 +27,8 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - virtual Status get_group_file(const std::string& table_id_, - const std::string& file_id_, - TableFileSchema& group_file_info_) override; + virtual Status GetTableFile(TableFileSchema& file_schema) override; + virtual Status update_group_file(TableFileSchema& group_file_) override; virtual Status get_group_files(const std::string& table_id_, diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 3923df7fb1..0e910433db 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -30,9 +30,7 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; - virtual Status get_group_file(const std::string& table_id_, - const std::string& file_id_, - TableFileSchema& group_file_info_) = 0; + virtual Status GetTableFile(TableFileSchema& file_schema) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0; virtual Status get_group_files(const std::string& table_id_, diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index b016b947e4..f56f1da2b2 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -85,7 +85,7 @@ TEST_F(MetaTest, table_file_TEST) { dates.push_back(table_file.date); status = impl_->DropPartitionsByDates(table_file.table_id, dates); ASSERT_TRUE(status.ok()); - status = impl_->get_group_file(table_file.table_id, table_file.file_id, table_file); + status = impl_->GetTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE); } @@ -127,7 +127,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.table_id, file.file_id, file); + status = impl.GetTableFile(file); ASSERT_TRUE(status.ok()); if (days[i] < days_num) { ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW); @@ -170,7 +170,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { int i = 0; for (auto file : files) { - status = impl.get_group_file(file.table_id, file.file_id, file); + status = impl.GetTableFile(file); ASSERT_TRUE(status.ok()); if (i < 5) { ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE); From 3852b830b0b178e55a9b83b7b71b1fcbee05686e Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:33:26 +0800 Subject: [PATCH 12/31] refactor(db): delete get_group_files Former-commit-id: 0503a8a92ab9687912c516390af1c8fe0e4cd96a --- cpp/src/db/DB.h | 3 --- cpp/src/db/DBImpl.cpp | 8 -------- cpp/src/db/DBImpl.h | 4 ---- cpp/src/db/DBMetaImpl.cpp | 7 ------- cpp/src/db/DBMetaImpl.h | 4 ---- cpp/src/db/Meta.h | 4 ---- 6 files changed, 30 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index b120927ef5..b0d7a31db1 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -24,9 +24,6 @@ public: virtual Status add_group(meta::TableSchema& table_schema_) = 0; virtual Status get_group(meta::TableSchema& table_schema_) = 0; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; - virtual Status get_group_files(const std::string& table_id_, - const int date_delta_, - meta::TableFilesSchema& group_files_info_) = 0; virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 2cbf6900c9..6e0df688b9 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -49,14 +49,6 @@ Status DBImpl::has_group(const std::string& table_id, bool& has_or_not) return _pMeta->HasTable(table_id, has_or_not); } -template -Status DBImpl::get_group_files(const std::string& table_id, - const int date_delta, - meta::TableFilesSchema& group_files_info) { - return _pMeta->get_group_files(table_id, date_delta, group_files_info); - -} - template Status DBImpl::add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 55741a5937..2303fefc57 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -37,10 +37,6 @@ public: virtual Status get_group(meta::TableSchema& table_schema) override; virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; - virtual Status get_group_files(const std::string& table_id_, - const int date_delta_, - meta::TableFilesSchema& group_files_info_) override; - virtual Status add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) override; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index f80e1281e1..24b12abb0b 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -431,13 +431,6 @@ Status DBMetaImpl::GetTableFile(TableFileSchema& file_schema) { return Status::OK(); } -Status DBMetaImpl::get_group_files(const std::string& table_id_, - const int date_delta_, - TableFilesSchema& group_files_info_) { - // PXU TODO - return Status::OK(); -} - // PXU TODO: Support Swap Status DBMetaImpl::archive_files() { auto& criterias = _options.archive_conf.GetCriterias(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index e2b5a5028e..db77534247 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -31,10 +31,6 @@ public: virtual Status update_group_file(TableFileSchema& group_file_) override; - virtual Status get_group_files(const std::string& table_id_, - const int date_delta_, - TableFilesSchema& group_files_info_) override; - virtual Status update_files(TableFilesSchema& files) override; virtual Status files_to_merge(const std::string& table_id, diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 0e910433db..00b4b664cb 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -33,10 +33,6 @@ public: virtual Status GetTableFile(TableFileSchema& file_schema) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0; - virtual Status get_group_files(const std::string& table_id_, - const int date_delta_, - TableFilesSchema& group_files_info_) = 0; - virtual Status update_files(TableFilesSchema& files) = 0; virtual Status files_to_search(const std::string& table_id, From ec4e83db3fa011e55616f744a834e82c7dacdbcf Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:38:55 +0800 Subject: [PATCH 13/31] refactor(db): UpdateTableFile Former-commit-id: e5b29fa1396d5a06c26a4917e15fed0082bbc7ae --- cpp/src/db/DBMetaImpl.cpp | 8 ++++---- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/MemManager.cpp | 2 +- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 16 ++++++++-------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 24b12abb0b..5ccc2d9f4a 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -538,13 +538,13 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { return discard_files_of_size(to_discard_size); } -Status DBMetaImpl::update_group_file(TableFileSchema& group_file) { - group_file.updated_time = utils::GetMicroSecTimeStamp(); +Status DBMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { + file_schema.updated_time = utils::GetMicroSecTimeStamp(); try { - ConnectorPtr->update(group_file); + ConnectorPtr->update(file_schema); } catch (std::exception & e) { LOG(DEBUG) << e.what(); - LOG(DEBUG) << "id= " << group_file.id << " file_id=" << group_file.file_id; + LOG(DEBUG) << "table_id= " << file_schema.table_id << " file_id=" << file_schema.file_id; throw e; } return Status::OK(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index db77534247..813b07c20f 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -29,7 +29,7 @@ public: virtual Status GetTableFile(TableFileSchema& file_schema) override; - virtual Status update_group_file(TableFileSchema& group_file_) override; + virtual Status UpdateTableFile(TableFileSchema& file_schema) override; virtual Status update_files(TableFilesSchema& files) override; diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 60553c7898..03ed3de1f3 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -55,7 +55,7 @@ Status MemVectors::serialize(std::string& table_id) { schema_.file_type = (size >= options_.index_trigger_size) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; - auto status = pMeta_->update_group_file(schema_); + auto status = pMeta_->UpdateTableFile(schema_); LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index") << " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M"; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 00b4b664cb..15d9b0e575 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -31,7 +31,7 @@ public: const DatesT& dates) = 0; virtual Status GetTableFile(TableFileSchema& file_schema) = 0; - virtual Status update_group_file(TableFileSchema& group_file_) = 0; + virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; virtual Status update_files(TableFilesSchema& files) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index f56f1da2b2..620b91139a 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -59,7 +59,7 @@ TEST_F(MetaTest, table_file_TEST) { auto new_file_type = meta::TableFileSchema::INDEX; table_file.file_type = new_file_type; - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type, new_file_type); @@ -76,7 +76,7 @@ TEST_F(MetaTest, table_file_TEST) { ASSERT_TRUE(status.ok()); table_file.date = meta::Meta::GetDateWithDelta(-2); - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2)); ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE); @@ -118,7 +118,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { table_file.file_type = meta::TableFileSchema::NEW; int day = rand() % (days_num*2); table_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000; - status = impl.update_group_file(table_file); + status = impl.UpdateTableFile(table_file); files.push_back(table_file); days.push_back(day); } @@ -162,7 +162,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { status = impl.CreateTableFile(table_file); table_file.file_type = meta::TableFileSchema::NEW; table_file.size = each_size * meta::G; - status = impl.update_group_file(table_file); + status = impl.UpdateTableFile(table_file); files.push_back(table_file); } @@ -201,25 +201,25 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { for (auto i=0; iCreateTableFile(table_file); table_file.file_type = meta::TableFileSchema::NEW; - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type = meta::TableFileSchema::RAW; - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type = meta::TableFileSchema::TO_INDEX; - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); } for (auto i=0; iCreateTableFile(table_file); table_file.file_type = meta::TableFileSchema::INDEX; - status = impl_->update_group_file(table_file); + status = impl_->UpdateTableFile(table_file); } meta::TableFilesSchema files; From edf33115e4f8cdd993155da1677094c76b0544fd Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:41:33 +0800 Subject: [PATCH 14/31] refactor(db): UpdateTableFiles Former-commit-id: 749cfe1d229a7f2edadd3cd913b96e0087be9cc5 --- cpp/src/db/DBImpl.cpp | 4 ++-- cpp/src/db/DBMetaImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 6e0df688b9..f71afbd02f 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -280,7 +280,7 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat } table_file.size = index_size; updated.push_back(table_file); - status = _pMeta->update_files(updated); + status = _pMeta->UpdateTableFiles(updated); LOG(DEBUG) << "New merged file " << table_file.file_id << " of size=" << index.PhysicalSize()/(1024*1024) << " M"; @@ -343,7 +343,7 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { to_remove.file_type = meta::TableFileSchema::TO_DELETE; meta::TableFilesSchema update_files = {to_remove, table_file}; - _pMeta->update_files(update_files); + _pMeta->UpdateTableFiles(update_files); LOG(DEBUG) << "New index file " << table_file.file_id << " of size " << index->PhysicalSize()/(1024*1024) << " M" diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 5ccc2d9f4a..ad2a700115 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -550,7 +550,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { return Status::OK(); } -Status DBMetaImpl::update_files(TableFilesSchema& files) { +Status DBMetaImpl::UpdateTableFiles(TableFilesSchema& files) { try { auto commited = ConnectorPtr->transaction([&] () mutable { for (auto& file : files) { diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 813b07c20f..d7c057828f 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -31,7 +31,7 @@ public: virtual Status UpdateTableFile(TableFileSchema& file_schema) override; - virtual Status update_files(TableFilesSchema& files) override; + virtual Status UpdateTableFiles(TableFilesSchema& files) override; virtual Status files_to_merge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 15d9b0e575..d7f0797cf6 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -33,7 +33,7 @@ public: virtual Status GetTableFile(TableFileSchema& file_schema) = 0; virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; - virtual Status update_files(TableFilesSchema& files) = 0; + virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; virtual Status files_to_search(const std::string& table_id, const DatesT& partition, From 09fdd0f8499d238425820a6a148f02bb237cc775 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:53:36 +0800 Subject: [PATCH 15/31] refactor(db): FilesToSearch Former-commit-id: dcb7f972ee13a0916129dfec5b05dad66025dfbf --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 27 ++++++++++++++------------- cpp/src/db/DBMetaImpl.h | 8 ++++---- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 2 +- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f71afbd02f..a0a01f8239 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -70,7 +70,7 @@ Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; - auto status = _pMeta->files_to_search(table_id, dates, files); + auto status = _pMeta->FilesToSearch(table_id, dates, files); if (!status.ok()) { return status; } LOG(DEBUG) << "Search DateT Size=" << files.size(); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index ad2a700115..baaf6c910a 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -303,7 +303,7 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { return Status::OK(); } -Status DBMetaImpl::files_to_search(const std::string &table_id, +Status DBMetaImpl::FilesToSearch(const std::string &table_id, const DatesT& partition, DatePartionedTableFilesSchema &files) { files.clear(); @@ -330,21 +330,22 @@ Status DBMetaImpl::files_to_search(const std::string &table_id, return status; } + TableFileSchema table_file; + for (auto& file : selected) { - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.table_id = std::get<1>(file); - group_file.file_id = std::get<2>(file); - group_file.file_type = std::get<3>(file); - group_file.size = std::get<4>(file); - group_file.date = std::get<5>(file); - group_file.dimension = table_schema.dimension; - GetGroupFilePath(group_file); - auto dateItr = files.find(group_file.date); + table_file.id = std::get<0>(file); + table_file.table_id = std::get<1>(file); + table_file.file_id = std::get<2>(file); + table_file.file_type = std::get<3>(file); + table_file.size = std::get<4>(file); + table_file.date = std::get<5>(file); + table_file.dimension = table_schema.dimension; + GetGroupFilePath(table_file); + auto dateItr = files.find(table_file.date); if (dateItr == files.end()) { - files[group_file.date] = TableFilesSchema(); + files[table_file.date] = TableFilesSchema(); } - files[group_file.date].push_back(group_file); + files[table_file.date].push_back(table_file); } } catch (std::exception & e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index d7c057828f..2a7c30730b 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -33,13 +33,13 @@ public: virtual Status UpdateTableFiles(TableFilesSchema& files) override; - virtual Status files_to_merge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; - - virtual Status files_to_search(const std::string& table_id, + virtual Status FilesToSearch(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) override; + virtual Status files_to_merge(const std::string& table_id, + DatePartionedTableFilesSchema& files) override; + virtual Status files_to_index(TableFilesSchema&) override; virtual Status archive_files() override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index d7f0797cf6..825e8bde6a 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -35,7 +35,7 @@ public: virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; - virtual Status files_to_search(const std::string& table_id, + virtual Status FilesToSearch(const std::string& table_id, const DatesT& partition, DatePartionedTableFilesSchema& files) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 620b91139a..42f5344e6d 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -238,7 +238,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatesT dates = {table_file.date}; - status = impl_->files_to_search(table_id, dates, dated_files); + status = impl_->FilesToSearch(table_id, dates, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); From 27892349313f028bd92d6838b6288038561152f3 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 18:56:42 +0800 Subject: [PATCH 16/31] refactor(db): FilesToMerge Former-commit-id: 9101dcd75943bc241094af92156805090bf8e408 --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 26 +++++++++++++------------- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a0a01f8239..c450f88d94 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -292,7 +292,7 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat template Status DBImpl::background_merge_files(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; - auto status = _pMeta->files_to_merge(table_id, raw_files); + auto status = _pMeta->FilesToMerge(table_id, raw_files); if (!status.ok()) { return status; } diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index baaf6c910a..35b3f2b529 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -355,7 +355,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::files_to_merge(const std::string& table_id, +Status DBMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); @@ -376,21 +376,21 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id, return status; } + TableFileSchema table_file; for (auto& file : selected) { - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.table_id = std::get<1>(file); - group_file.file_id = std::get<2>(file); - group_file.file_type = std::get<3>(file); - group_file.size = std::get<4>(file); - group_file.date = std::get<5>(file); - group_file.dimension = table_schema.dimension; - GetGroupFilePath(group_file); - auto dateItr = files.find(group_file.date); + table_file.id = std::get<0>(file); + table_file.table_id = std::get<1>(file); + table_file.file_id = std::get<2>(file); + table_file.file_type = std::get<3>(file); + table_file.size = std::get<4>(file); + table_file.date = std::get<5>(file); + table_file.dimension = table_schema.dimension; + GetGroupFilePath(table_file); + auto dateItr = files.find(table_file.date); if (dateItr == files.end()) { - files[group_file.date] = TableFilesSchema(); + files[table_file.date] = TableFilesSchema(); } - files[group_file.date].push_back(group_file); + files[table_file.date].push_back(table_file); } } catch (std::exception & e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 2a7c30730b..3e888ccd57 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -37,7 +37,7 @@ public: const DatesT& partition, DatePartionedTableFilesSchema& files) override; - virtual Status files_to_merge(const std::string& table_id, + virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; virtual Status files_to_index(TableFilesSchema&) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 825e8bde6a..3016ffa9bf 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -39,7 +39,7 @@ public: const DatesT& partition, DatePartionedTableFilesSchema& files) = 0; - virtual Status files_to_merge(const std::string& table_id, + virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; virtual Status size(long& result) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 42f5344e6d..7e4e2bf13d 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -229,7 +229,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->files_to_merge(group.table_id, dated_files); + status = impl_->FilesToMerge(group.table_id, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt); From de8811ae9a504a4458543dc71514824d0cebd5a4 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:00:47 +0800 Subject: [PATCH 17/31] refactor(db): Size Former-commit-id: 9dc93505e1378a1e8b14750158c2f8b9b2f840ca --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 4 ++-- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c450f88d94..f3978447f5 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -413,7 +413,7 @@ Status DBImpl::count(const std::string& table_id, long& result) { template Status DBImpl::size(long& result) { - return _pMeta->size(result); + return _pMeta->Size(result); } template diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 35b3f2b529..4210822d73 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -462,7 +462,7 @@ Status DBMetaImpl::archive_files() { } if (criteria == "disk") { long sum = 0; - size(sum); + Size(sum); // PXU TODO: refactor size auto to_delete = (sum - limit*G); @@ -473,7 +473,7 @@ Status DBMetaImpl::archive_files() { return Status::OK(); } -Status DBMetaImpl::size(long& result) { +Status DBMetaImpl::Size(long& result) { result = 0; try { auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)), diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 3e888ccd57..fbc3aa1d79 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -44,7 +44,7 @@ public: virtual Status archive_files() override; - virtual Status size(long& result) override; + virtual Status Size(long& result) override; virtual Status cleanup() override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 3016ffa9bf..060d8c772c 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -42,7 +42,7 @@ public: virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; - virtual Status size(long& result) = 0; + virtual Status Size(long& result) = 0; virtual Status archive_files() = 0; From 7fe5346f14d8383c8ef265547a200da727a7fc75 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:06:42 +0800 Subject: [PATCH 18/31] refactor(db): Archive Former-commit-id: 21f153c1a57ad4bb706f6b177a10d45faf88ed99 --- cpp/src/db/DBImpl.cpp | 4 ++-- cpp/src/db/DBMetaImpl.cpp | 16 ++++++++-------- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f3978447f5..057573be04 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -312,7 +312,7 @@ Status DBImpl::background_merge_files(const std::string& table_id) { merge_files(table_id, kv.first, kv.second); } - _pMeta->archive_files(); + _pMeta->Archive(); try_build_index(); @@ -350,7 +350,7 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { << " from file " << to_remove.file_id; index->Cache(); - _pMeta->archive_files(); + _pMeta->Archive(); return Status::OK(); } diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 4210822d73..4871d5a51f 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -433,7 +433,7 @@ Status DBMetaImpl::GetTableFile(TableFileSchema& file_schema) { } // PXU TODO: Support Swap -Status DBMetaImpl::archive_files() { +Status DBMetaImpl::Archive() { auto& criterias = _options.archive_conf.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); @@ -496,7 +496,7 @@ Status DBMetaImpl::Size(long& result) { } Status DBMetaImpl::discard_files_of_size(long to_discard_size) { - LOG(DEBUG) << "Abort to discard size=" << to_discard_size; + LOG(DEBUG) << "About to discard size=" << to_discard_size; if (to_discard_size <= 0) { return Status::OK(); } @@ -507,15 +507,15 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { order_by(&TableFileSchema::id), limit(10)); std::vector ids; + TableFileSchema table_file; for (auto& file : selected) { if (to_discard_size <= 0) break; - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.size = std::get<1>(file); - ids.push_back(group_file.id); - LOG(DEBUG) << "Discard group_file.id=" << group_file.id << " group_file.size=" << group_file.size; - to_discard_size -= group_file.size; + table_file.id = std::get<0>(file); + table_file.size = std::get<1>(file); + ids.push_back(table_file.id); + LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id << " table_file.size=" << table_file.size; + to_discard_size -= table_file.size; } if (ids.size() == 0) { diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index fbc3aa1d79..8832c30a52 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -42,7 +42,7 @@ public: virtual Status files_to_index(TableFilesSchema&) override; - virtual Status archive_files() override; + virtual Status Archive() override; virtual Status Size(long& result) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 060d8c772c..6a0993a9a6 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -44,7 +44,7 @@ public: virtual Status Size(long& result) = 0; - virtual Status archive_files() = 0; + virtual Status Archive() = 0; virtual Status files_to_index(TableFilesSchema&) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 7e4e2bf13d..8051d2ebcc 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -123,7 +123,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { days.push_back(day); } - impl.archive_files(); + impl.Archive(); int i = 0; for (auto file : files) { @@ -166,7 +166,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { files.push_back(table_file); } - impl.archive_files(); + impl.Archive(); int i = 0; for (auto file : files) { From 874eeb8f7a13c0cca0e6af6040ae9fe7c675f4c0 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:10:49 +0800 Subject: [PATCH 19/31] refactor(db): FilesToIndex Former-commit-id: fe5bd8ca372d0a4d5bf257f3f10853179d2356ce --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 28 ++++++++++++++-------------- cpp/src/db/DBMetaImpl.h | 2 +- cpp/src/db/Meta.h | 2 +- cpp/unittest/db/meta_tests.cpp | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 057573be04..57cd886394 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -360,7 +360,7 @@ void DBImpl::background_build_index() { std::lock_guard lock(build_index_mutex_); assert(bg_build_index_started_); meta::TableFilesSchema to_index_files; - _pMeta->files_to_index(to_index_files); + _pMeta->FilesToIndex(to_index_files); Status status; for (auto& file : to_index_files) { /* LOG(DEBUG) << "Buiding index for " << file.location; */ diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 4871d5a51f..1993992f24 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -259,7 +259,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) { return Status::OK(); } -Status DBMetaImpl::files_to_index(TableFilesSchema& files) { +Status DBMetaImpl::FilesToIndex(TableFilesSchema& files) { files.clear(); try { @@ -272,28 +272,28 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) { where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX)); std::map groups; + TableFileSchema table_file; for (auto& file : selected) { - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.table_id = std::get<1>(file); - group_file.file_id = std::get<2>(file); - group_file.file_type = std::get<3>(file); - group_file.size = std::get<4>(file); - group_file.date = std::get<5>(file); - GetGroupFilePath(group_file); - auto groupItr = groups.find(group_file.table_id); + table_file.id = std::get<0>(file); + table_file.table_id = std::get<1>(file); + table_file.file_id = std::get<2>(file); + table_file.file_type = std::get<3>(file); + table_file.size = std::get<4>(file); + table_file.date = std::get<5>(file); + GetGroupFilePath(table_file); + auto groupItr = groups.find(table_file.table_id); if (groupItr == groups.end()) { TableSchema table_schema; - table_schema.table_id = group_file.table_id; + table_schema.table_id = table_file.table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - groups[group_file.table_id] = table_schema; + groups[table_file.table_id] = table_schema; } - group_file.dimension = groups[group_file.table_id].dimension; - files.push_back(group_file); + table_file.dimension = groups[table_file.table_id].dimension; + files.push_back(table_file); } } catch (std::exception & e) { LOG(DEBUG) << e.what(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 8832c30a52..9993ace9a5 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -40,7 +40,7 @@ public: virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - virtual Status files_to_index(TableFilesSchema&) override; + virtual Status FilesToIndex(TableFilesSchema&) override; virtual Status Archive() override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 6a0993a9a6..d5206d6503 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -46,7 +46,7 @@ public: virtual Status Archive() = 0; - virtual Status files_to_index(TableFilesSchema&) = 0; + virtual Status FilesToIndex(TableFilesSchema&) = 0; virtual Status cleanup() = 0; virtual Status cleanup_ttl_files(uint16_t) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 8051d2ebcc..1e8498121b 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -224,7 +224,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { meta::TableFilesSchema files; - status = impl_->files_to_index(files); + status = impl_->FilesToIndex(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); @@ -233,7 +233,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt); - status = impl_->files_to_index(files); + status = impl_->FilesToIndex(files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), to_index_files_cnt); From ca33fbbe143269aa80a030452b6de5d0940a9830 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:22:58 +0800 Subject: [PATCH 20/31] refactor(db): DropAll -Count CleanUp CleanUpFilesWithTTL Former-commit-id: b718bb89945651de19a1263eaf64eab22d11eb6c --- cpp/src/db/DB.h | 2 -- cpp/src/db/DBImpl.cpp | 9 ++--- cpp/src/db/DBImpl.h | 2 -- cpp/src/db/DBMetaImpl.cpp | 60 +++++++++++++++++----------------- cpp/src/db/DBMetaImpl.h | 8 ++--- cpp/src/db/Meta.h | 8 ++--- cpp/unittest/db/db_tests.cpp | 4 +-- cpp/unittest/db/meta_tests.cpp | 4 +-- cpp/unittest/db/utils.cpp | 2 +- 9 files changed, 45 insertions(+), 54 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index b0d7a31db1..c6859b35c6 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -38,8 +38,6 @@ public: virtual Status drop_all() = 0; - virtual Status count(const std::string& table_id, long& result) = 0; - DB() = default; DB(const DB&) = delete; DB& operator=(const DB&) = delete; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 57cd886394..45dc813d4b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -316,7 +316,7 @@ Status DBImpl::background_merge_files(const std::string& table_id) { try_build_index(); - _pMeta->cleanup_ttl_files(1); + _pMeta->CleanUpFilesWithTTL(1); return Status::OK(); } @@ -403,12 +403,7 @@ void DBImpl::background_compaction() { template Status DBImpl::drop_all() { - return _pMeta->drop_all(); -} - -template -Status DBImpl::count(const std::string& table_id, long& result) { - return _pMeta->count(table_id, result); + return _pMeta->DropAll(); } template diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 2303fefc57..523922d3d3 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -48,8 +48,6 @@ public: virtual Status drop_all() override; - virtual Status count(const std::string& table_id, long& result) override; - virtual Status size(long& result) override; virtual ~DBImpl(); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 1993992f24..325ba2486b 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -105,7 +105,7 @@ Status DBMetaImpl::initialize() { ConnectorPtr->open_forever(); // thread safe option ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log - cleanup(); + CleanUp(); return Status::OK(); } @@ -570,7 +570,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema& files) { return Status::OK(); } -Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { +Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, @@ -583,21 +583,21 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { c(&TableFileSchema::updated_time) > now - seconds*US_PS)); TableFilesSchema updated; + TableFileSchema table_file; for (auto& file : selected) { - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.table_id = std::get<1>(file); - group_file.file_id = std::get<2>(file); - group_file.file_type = std::get<3>(file); - group_file.size = std::get<4>(file); - group_file.date = std::get<5>(file); - GetGroupFilePath(group_file); - if (group_file.file_type == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(group_file.location); + table_file.id = std::get<0>(file); + table_file.table_id = std::get<1>(file); + table_file.file_id = std::get<2>(file); + table_file.file_type = std::get<3>(file); + table_file.size = std::get<4>(file); + table_file.date = std::get<5>(file); + GetGroupFilePath(table_file); + if (table_file.file_type == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location); } - ConnectorPtr->remove(group_file.id); - /* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */ + ConnectorPtr->remove(table_file.id); + /* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */ } } catch (std::exception & e) { LOG(DEBUG) << e.what(); @@ -607,7 +607,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { return Status::OK(); } -Status DBMetaImpl::cleanup() { +Status DBMetaImpl::CleanUp() { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, &TableFileSchema::table_id, @@ -619,21 +619,21 @@ Status DBMetaImpl::cleanup() { c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW)); TableFilesSchema updated; + TableFileSchema table_file; for (auto& file : selected) { - TableFileSchema group_file; - group_file.id = std::get<0>(file); - group_file.table_id = std::get<1>(file); - group_file.file_id = std::get<2>(file); - group_file.file_type = std::get<3>(file); - group_file.size = std::get<4>(file); - group_file.date = std::get<5>(file); - GetGroupFilePath(group_file); - if (group_file.file_type == TableFileSchema::TO_DELETE) { - boost::filesystem::remove(group_file.location); + table_file.id = std::get<0>(file); + table_file.table_id = std::get<1>(file); + table_file.file_id = std::get<2>(file); + table_file.file_type = std::get<3>(file); + table_file.size = std::get<4>(file); + table_file.date = std::get<5>(file); + GetGroupFilePath(table_file); + if (table_file.file_type == TableFileSchema::TO_DELETE) { + boost::filesystem::remove(table_file.location); } - ConnectorPtr->remove(group_file.id); - /* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */ + ConnectorPtr->remove(table_file.id); + /* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */ } } catch (std::exception & e) { LOG(DEBUG) << e.what(); @@ -643,7 +643,7 @@ Status DBMetaImpl::cleanup() { return Status::OK(); } -Status DBMetaImpl::count(const std::string& table_id, long& result) { +Status DBMetaImpl::Count(const std::string& table_id, long& result) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, @@ -674,7 +674,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) { return Status::OK(); } -Status DBMetaImpl::drop_all() { +Status DBMetaImpl::DropAll() { if (boost::filesystem::is_directory(_options.path)) { boost::filesystem::remove_all(_options.path); } @@ -682,7 +682,7 @@ Status DBMetaImpl::drop_all() { } DBMetaImpl::~DBMetaImpl() { - cleanup(); + CleanUp(); } } // namespace meta diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 9993ace9a5..ea259a7f58 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -46,13 +46,13 @@ public: virtual Status Size(long& result) override; - virtual Status cleanup() override; + virtual Status CleanUp() override; - virtual Status cleanup_ttl_files(uint16_t seconds) override; + virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status drop_all() override; + virtual Status DropAll() override; - virtual Status count(const std::string& table_id, long& result) override; + virtual Status Count(const std::string& table_id, long& result) override; virtual ~DBMetaImpl(); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index d5206d6503..463f5a66c2 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -48,12 +48,12 @@ public: virtual Status FilesToIndex(TableFilesSchema&) = 0; - virtual Status cleanup() = 0; - virtual Status cleanup_ttl_files(uint16_t) = 0; + virtual Status CleanUp() = 0; + virtual Status CleanUpFilesWithTTL(uint16_t) = 0; - virtual Status drop_all() = 0; + virtual Status DropAll() = 0; - virtual Status count(const std::string& table_id, long& result) = 0; + virtual Status Count(const std::string& table_id, long& result) = 0; static DateT GetDate(const std::time_t& t, int day_delta = 0); static DateT GetDate(); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 381fa28e95..a5d5bcf1fc 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -152,12 +152,12 @@ TEST_F(DBTest, DB_TEST) { for (auto j=0; j<10; ++j) { ss.str(""); - db_->count(group_name, count); + db_->size(count); prev_count = count; START_TIMER; stat = db_->search(group_name, k, qb, qxb, results); - ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/engine::meta::M << " M"; + ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); ASSERT_STATS(stat); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 1e8498121b..abd03d3057 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -137,7 +137,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { i++; } - impl.drop_all(); + impl.DropAll(); } TEST_F(MetaTest, ARCHIVE_TEST_DISK) { @@ -180,7 +180,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { ++i; } - impl.drop_all(); + impl.DropAll(); } TEST_F(MetaTest, TABLE_FILES_TEST) { diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 09428427fe..aa3ea560f3 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -59,5 +59,5 @@ void MetaTest::SetUp() { } void MetaTest::TearDown() { - impl_->drop_all(); + impl_->DropAll(); } From 2eff64def114eb8882cc86f0e41d14c1f7dc392a Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:31:29 +0800 Subject: [PATCH 21/31] refactor(db): refactor for DBMetaImpl Former-commit-id: 9ced33145335090265c53344b4ff46a977eec356 --- cpp/src/db/DBMetaImpl.cpp | 76 +++++++++++++++++++-------------------- cpp/src/db/DBMetaImpl.h | 14 ++++---- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 325ba2486b..93faadcacb 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -27,13 +27,13 @@ using namespace sqlite_orm; inline auto StoragePrototype(const std::string& path) { return make_storage(path, - make_table("Group", + make_table("Table", make_column("id", &TableSchema::id, primary_key()), make_column("table_id", &TableSchema::table_id, unique()), make_column("dimension", &TableSchema::dimension), make_column("created_on", &TableSchema::created_on), make_column("files_cnt", &TableSchema::files_cnt, default_value(0))), - make_table("GroupFile", + make_table("TableFile", make_column("id", &TableFileSchema::id, primary_key()), make_column("table_id", &TableFileSchema::table_id), make_column("file_id", &TableFileSchema::file_id), @@ -49,27 +49,27 @@ inline auto StoragePrototype(const std::string& path) { using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -std::string DBMetaImpl::GetGroupPath(const std::string& table_id) { - return _options.path + "/tables/" + table_id; +std::string DBMetaImpl::GetTablePath(const std::string& table_id) { + return options_.path + "/tables/" + table_id; } -std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) { +std::string DBMetaImpl::GetTableDatePartitionPath(const std::string& table_id, DateT& date) { std::stringstream ss; - ss << GetGroupPath(table_id) << "/" << date; + ss << GetTablePath(table_id) << "/" << date; return ss.str(); } -void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) { +void DBMetaImpl::GetTableFilePath(TableFileSchema& group_file) { if (group_file.date == EmptyDate) { group_file.date = Meta::GetDate(); } std::stringstream ss; - ss << GetGroupDatePartitionPath(group_file.table_id, group_file.date) + ss << GetTableDatePartitionPath(group_file.table_id, group_file.date) << "/" << group_file.file_id; group_file.location = ss.str(); } -Status DBMetaImpl::NextGroupId(std::string& table_id) { +Status DBMetaImpl::NextTableId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.getNextIDNumber(); @@ -86,20 +86,20 @@ Status DBMetaImpl::NextFileId(std::string& file_id) { } DBMetaImpl::DBMetaImpl(const DBMetaOptions& options_) - : _options(options_) { - initialize(); + : options_(options_) { + Initialize(); } -Status DBMetaImpl::initialize() { - if (!boost::filesystem::is_directory(_options.path)) { - auto ret = boost::filesystem::create_directory(_options.path); +Status DBMetaImpl::Initialize() { + if (!boost::filesystem::is_directory(options_.path)) { + auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { - LOG(ERROR) << "Create directory " << _options.path << " Error"; + LOG(ERROR) << "Create directory " << options_.path << " Error"; } assert(ret); } - ConnectorPtr = std::make_unique(StoragePrototype(_options.path+"/meta.sqlite")); + ConnectorPtr = std::make_unique(StoragePrototype(options_.path+"/meta.sqlite")); ConnectorPtr->sync_schema(); ConnectorPtr->open_forever(); // thread safe option @@ -150,7 +150,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string& table_id, Status DBMetaImpl::CreateTable(TableSchema& table_schema) { if (table_schema.table_id == "") { - NextGroupId(table_schema.table_id); + NextTableId(table_schema.table_id); } table_schema.files_cnt = 0; table_schema.id = -1; @@ -161,11 +161,11 @@ Status DBMetaImpl::CreateTable(TableSchema& table_schema) { auto id = ConnectorPtr->insert(table_schema); table_schema.id = id; } catch (...) { - return Status::DBTransactionError("Add Group Error"); + return Status::DBTransactionError("Add Table Error"); } } - auto group_path = GetGroupPath(table_schema.table_id); + auto group_path = GetTablePath(table_schema.table_id); if (!boost::filesystem::is_directory(group_path)) { auto ret = boost::filesystem::create_directories(group_path); @@ -191,7 +191,7 @@ Status DBMetaImpl::DescribeTable(TableSchema& table_schema) { table_schema.files_cnt = std::get<2>(groups[0]); table_schema.dimension = std::get<3>(groups[0]); } else { - return Status::NotFound("Group " + table_schema.table_id + " not found"); + return Status::NotFound("Table " + table_schema.table_id + " not found"); } } catch (std::exception &e) { LOG(DEBUG) << e.what(); @@ -235,7 +235,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.size = 0; file_schema.created_on = utils::GetMicroSecTimeStamp(); file_schema.updated_time = file_schema.created_on; - GetGroupFilePath(file_schema); + GetTableFilePath(file_schema); { try { @@ -246,7 +246,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema& file_schema) { } } - auto partition_path = GetGroupDatePartitionPath(file_schema.table_id, file_schema.date); + auto partition_path = GetTableDatePartitionPath(file_schema.table_id, file_schema.date); if (!boost::filesystem::is_directory(partition_path)) { auto ret = boost::filesystem::create_directory(partition_path); @@ -281,7 +281,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema& files) { table_file.file_type = std::get<3>(file); table_file.size = std::get<4>(file); table_file.date = std::get<5>(file); - GetGroupFilePath(table_file); + GetTableFilePath(table_file); auto groupItr = groups.find(table_file.table_id); if (groupItr == groups.end()) { TableSchema table_schema; @@ -340,7 +340,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, table_file.size = std::get<4>(file); table_file.date = std::get<5>(file); table_file.dimension = table_schema.dimension; - GetGroupFilePath(table_file); + GetTableFilePath(table_file); auto dateItr = files.find(table_file.date); if (dateItr == files.end()) { files[table_file.date] = TableFilesSchema(); @@ -385,7 +385,7 @@ Status DBMetaImpl::FilesToMerge(const std::string& table_id, table_file.size = std::get<4>(file); table_file.date = std::get<5>(file); table_file.dimension = table_schema.dimension; - GetGroupFilePath(table_file); + GetTableFilePath(table_file); auto dateItr = files.find(table_file.date); if (dateItr == files.end()) { files[table_file.date] = TableFilesSchema(); @@ -434,7 +434,7 @@ Status DBMetaImpl::GetTableFile(TableFileSchema& file_schema) { // PXU TODO: Support Swap Status DBMetaImpl::Archive() { - auto& criterias = _options.archive_conf.GetCriterias(); + auto& criterias = options_.archive_conf.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); } @@ -464,9 +464,8 @@ Status DBMetaImpl::Archive() { long sum = 0; Size(sum); - // PXU TODO: refactor size auto to_delete = (sum - limit*G); - discard_files_of_size(to_delete); + DiscardFiles(to_delete); } } @@ -495,17 +494,18 @@ Status DBMetaImpl::Size(long& result) { return Status::OK(); } -Status DBMetaImpl::discard_files_of_size(long to_discard_size) { +Status DBMetaImpl::DiscardFiles(long to_discard_size) { LOG(DEBUG) << "About to discard size=" << to_discard_size; if (to_discard_size <= 0) { return Status::OK(); } try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, - &TableFileSchema::size), - where(c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE), - order_by(&TableFileSchema::id), - limit(10)); + &TableFileSchema::size), + where(c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE), + order_by(&TableFileSchema::id), + limit(10)); + std::vector ids; TableFileSchema table_file; @@ -536,7 +536,7 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) { } - return discard_files_of_size(to_discard_size); + return DiscardFiles(to_discard_size); } Status DBMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { @@ -592,7 +592,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { table_file.file_type = std::get<3>(file); table_file.size = std::get<4>(file); table_file.date = std::get<5>(file); - GetGroupFilePath(table_file); + GetTableFilePath(table_file); if (table_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(table_file.location); } @@ -628,7 +628,7 @@ Status DBMetaImpl::CleanUp() { table_file.file_type = std::get<3>(file); table_file.size = std::get<4>(file); table_file.date = std::get<5>(file); - GetGroupFilePath(table_file); + GetTableFilePath(table_file); if (table_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(table_file.location); } @@ -675,8 +675,8 @@ Status DBMetaImpl::Count(const std::string& table_id, long& result) { } Status DBMetaImpl::DropAll() { - if (boost::filesystem::is_directory(_options.path)) { - boost::filesystem::remove_all(_options.path); + if (boost::filesystem::is_directory(options_.path)) { + boost::filesystem::remove_all(options_.path); } return Status::OK(); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index ea259a7f58..9710a3be90 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -58,14 +58,14 @@ public: private: Status NextFileId(std::string& file_id); - Status NextGroupId(std::string& table_id); - Status discard_files_of_size(long to_discard_size); - std::string GetGroupPath(const std::string& table_id); - std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date); - void GetGroupFilePath(TableFileSchema& group_file); - Status initialize(); + Status NextTableId(std::string& table_id); + Status DiscardFiles(long to_discard_size); + std::string GetTablePath(const std::string& table_id); + std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); + void GetTableFilePath(TableFileSchema& group_file); + Status Initialize(); - const DBMetaOptions _options; + const DBMetaOptions options_; }; // DBMetaImpl } // namespace meta From fbeddc2ad54d2ad7fbe9b29883ad677fa51bffa1 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:39:12 +0800 Subject: [PATCH 22/31] refactor(db): refactor for execution engine Former-commit-id: 5bf686bae2669ecfe6c4ba722a001c6303bb2f43 --- cpp/src/db/ExecutionEngine.cpp | 3 ++- cpp/src/db/ExecutionEngine.h | 4 ++-- cpp/src/db/FaissExecutionEngine.h | 10 ++++++---- ...issExecutionEngine.cpp => FaissExecutionEngine.inl} | 9 +++------ 4 files changed, 13 insertions(+), 13 deletions(-) rename cpp/src/db/{FaissExecutionEngine.cpp => FaissExecutionEngine.inl} (98%) diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/ExecutionEngine.cpp index b689e6cfed..6eb04bb788 100644 --- a/cpp/src/db/ExecutionEngine.cpp +++ b/cpp/src/db/ExecutionEngine.cpp @@ -3,9 +3,10 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#include #include "ExecutionEngine.h" +#include + namespace zilliz { namespace vecwise { namespace engine { diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index 0b7af1b53e..6ad91959e8 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -5,11 +5,11 @@ ******************************************************************************/ #pragma once +#include "Status.h" + #include #include -#include "Status.h" - namespace zilliz { namespace vecwise { namespace engine { diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index 739e09efca..2915acc0eb 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -5,11 +5,11 @@ ******************************************************************************/ #pragma once +#include "ExecutionEngine.h" + #include #include -#include "ExecutionEngine.h" - namespace faiss { class Index; } @@ -22,7 +22,7 @@ namespace engine { template class FaissExecutionEngine : public ExecutionEngine> { public: - typedef std::shared_ptr> Ptr; + using Ptr = std::shared_ptr>; FaissExecutionEngine(uint16_t dimension, const std::string& location); FaissExecutionEngine(std::shared_ptr index, const std::string& location); @@ -53,7 +53,9 @@ public: Ptr BuildIndex(const std::string&); Status Cache(); + protected: + std::shared_ptr pIndex_; std::string location_; }; @@ -63,4 +65,4 @@ protected: } // namespace vecwise } // namespace zilliz -#include "FaissExecutionEngine.cpp" +#include "FaissExecutionEngine.inl" diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.inl similarity index 98% rename from cpp/src/db/FaissExecutionEngine.cpp rename to cpp/src/db/FaissExecutionEngine.inl index c2165c948d..961090f893 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.inl @@ -3,8 +3,9 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#ifndef FAISSEXECUTIONENGINE_CPP__ -#define FAISSEXECUTIONENGINE_CPP__ +#pragma once + +#include "FaissExecutionEngine.h" #include #include @@ -15,8 +16,6 @@ #include #include -#include "FaissExecutionEngine.h" - namespace zilliz { namespace vecwise { namespace engine { @@ -135,5 +134,3 @@ Status FaissExecutionEngine::Cache() { } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif From ce329b4304c515ab636306b7a0f498d42a3b803d Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:54:28 +0800 Subject: [PATCH 23/31] refactor(db): Env.cpp Env.h Former-commit-id: 5d774124457d4136dc8c0e4469ac6950c424de09 --- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/Env.cpp | 64 +++++++++++++++++++++---------------------- cpp/src/db/Env.h | 25 ++++++++--------- 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 45dc813d4b..90f0bf5c5d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -219,7 +219,7 @@ void DBImpl::try_schedule_compaction() { if (!_bg_error.ok()) return; _bg_compaction_scheduled = true; - _env->schedule(&DBImpl::BGWork, this); + _env->Schedule(&DBImpl::BGWork, this); } template diff --git a/cpp/src/db/Env.cpp b/cpp/src/db/Env.cpp index f3a4b64810..5c5bcdb52a 100644 --- a/cpp/src/db/Env.cpp +++ b/cpp/src/db/Env.cpp @@ -13,66 +13,66 @@ namespace vecwise { namespace engine { Env::Env() - : _bg_work_started(false), - _shutting_down(false) { + : bg_work_started_(false), + shutting_down_(false) { } -void Env::schedule(void (*function_)(void* arg_), void* arg_) { - std::unique_lock lock(_bg_work_mutex); - if (_shutting_down) return; +void Env::Schedule(void (*function)(void* arg), void* arg) { + std::unique_lock lock(bg_work_mutex_); + if (shutting_down_) return; - if (!_bg_work_started) { - _bg_work_started = true; + if (!bg_work_started_) { + bg_work_started_ = true; std::thread bg_thread(Env::BackgroundThreadEntryPoint, this); bg_thread.detach(); } - if (_bg_work_queue.empty()) { - _bg_work_cv.notify_one(); + if (bg_work_queue_.empty()) { + bg_work_cv_.notify_one(); } - _bg_work_queue.emplace(function_, arg_); + bg_work_queue_.emplace(function, arg); } -void Env::backgroud_thread_main() { - while (!_shutting_down) { - std::unique_lock lock(_bg_work_mutex); - while (_bg_work_queue.empty() && !_shutting_down) { - _bg_work_cv.wait(lock); +void Env::BackgroundThreadMain() { + while (!shutting_down_) { + std::unique_lock lock(bg_work_mutex_); + while (bg_work_queue_.empty() && !shutting_down_) { + bg_work_cv_.wait(lock); } - if (_shutting_down) break; + if (shutting_down_) break; - assert(!_bg_work_queue.empty()); - auto bg_function = _bg_work_queue.front()._function; - void* bg_arg = _bg_work_queue.front()._arg; - _bg_work_queue.pop(); + assert(!bg_work_queue_.empty()); + auto bg_function = bg_work_queue_.front().function_; + void* bg_arg = bg_work_queue_.front().arg_; + bg_work_queue_.pop(); lock.unlock(); bg_function(bg_arg); } - std::unique_lock lock(_bg_work_mutex); - _bg_work_started = false; - _bg_work_cv.notify_all(); + std::unique_lock lock(bg_work_mutex_); + bg_work_started_ = false; + bg_work_cv_.notify_all(); } void Env::Stop() { { - std::unique_lock lock(_bg_work_mutex); - if (_shutting_down || !_bg_work_started) return; + std::unique_lock lock(bg_work_mutex_); + if (shutting_down_ || !bg_work_started_) return; } - _shutting_down = true; + shutting_down_ = true; { - std::unique_lock lock(_bg_work_mutex); - if (_bg_work_queue.empty()) { - _bg_work_cv.notify_one(); + std::unique_lock lock(bg_work_mutex_); + if (bg_work_queue_.empty()) { + bg_work_cv_.notify_one(); } - while (_bg_work_started) { - _bg_work_cv.wait(lock); + while (bg_work_started_) { + bg_work_cv_.wait(lock); } } - _shutting_down = false; + shutting_down_ = false; } Env::~Env() {} diff --git a/cpp/src/db/Env.h b/cpp/src/db/Env.h index 9eed68cddb..28ecaaacaf 100644 --- a/cpp/src/db/Env.h +++ b/cpp/src/db/Env.h @@ -22,7 +22,7 @@ public: Env(const Env&) = delete; Env& operator=(const Env&) = delete; - void schedule(void (*function_)(void* arg_), void* arg_); + void Schedule(void (*function)(void* arg), void* arg); virtual void Stop(); @@ -31,25 +31,24 @@ public: static Env* Default(); protected: - void backgroud_thread_main(); + void BackgroundThreadMain(); static void BackgroundThreadEntryPoint(Env* env) { - env->backgroud_thread_main(); + env->BackgroundThreadMain(); } struct BGWork { - explicit BGWork(void (*function_)(void*), void* arg_) - : _function(function_), _arg(arg_) {} + explicit BGWork(void (*function)(void*), void* arg) + : function_(function), arg_(arg) {} - void (* const _function)(void*); - void* const _arg; + void (* const function_)(void*); + void* const arg_; }; - std::mutex _bg_work_mutex; - std::condition_variable _bg_work_cv; - std::queue _bg_work_queue; - bool _bg_work_started; - std::atomic _shutting_down; - + std::mutex bg_work_mutex_; + std::condition_variable bg_work_cv_; + std::queue bg_work_queue_; + bool bg_work_started_; + std::atomic shutting_down_; }; // Env } // namespace engine From 73ffe9f7966c0546d1db9a9ee1fdfda13eba6bc8 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 19:58:10 +0800 Subject: [PATCH 24/31] refactor(db): include files seq Former-commit-id: 1236730a49eb3ab3611e001aee95a6a28173f4ca --- cpp/src/db/DBMetaImpl.cpp | 9 ++++----- cpp/src/db/Meta.cpp | 3 ++- cpp/src/db/Meta.h | 7 ++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 93faadcacb..f6cc3a0ecd 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -3,6 +3,10 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ +#include "DBMetaImpl.h" +#include "IDGenerator.h" +#include "Utils.h" +#include "MetaConsts.h" #include #include @@ -13,11 +17,6 @@ #include #include -#include "DBMetaImpl.h" -#include "IDGenerator.h" -#include "Utils.h" -#include "MetaConsts.h" - namespace zilliz { namespace vecwise { namespace engine { diff --git a/cpp/src/db/Meta.cpp b/cpp/src/db/Meta.cpp index 3d272e0637..f0539ae1be 100644 --- a/cpp/src/db/Meta.cpp +++ b/cpp/src/db/Meta.cpp @@ -3,9 +3,10 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ +#include "Meta.h" + #include #include -#include "Meta.h" namespace zilliz { namespace vecwise { diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 463f5a66c2..b67191739f 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -4,14 +4,15 @@ * Proprietary and confidential. ******************************************************************************/ #pragma once -#include -#include -#include #include "MetaTypes.h" #include "Options.h" #include "Status.h" +#include +#include +#include + namespace zilliz { namespace vecwise { namespace engine { From 541692c227cde3f2630b6f09f9b3c18c3e597420 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 20:15:16 +0800 Subject: [PATCH 25/31] refactor(db): include file seq and cpp->inl Former-commit-id: 970bb6ef009b1806a4f60f6d1f5d62e0aeb1ca5e --- cpp/src/db/DB.cpp | 2 -- cpp/src/db/DBImpl.h | 12 ++++++------ cpp/src/db/{DBImpl.cpp => DBImpl.inl} | 13 +++++-------- cpp/src/db/DBMetaImpl.cpp | 4 ++-- cpp/src/db/Factories.cpp | 11 +++++------ cpp/src/db/Factories.h | 6 +++--- cpp/src/db/IDGenerator.cpp | 18 ++++++++---------- cpp/src/db/IDGenerator.h | 14 ++++++++------ cpp/src/db/MemManager.h | 10 +++++----- cpp/src/db/{MemManager.cpp => MemManager.inl} | 16 ++++++---------- cpp/src/db/Utils.cpp | 2 +- 11 files changed, 49 insertions(+), 59 deletions(-) rename cpp/src/db/{DBImpl.cpp => DBImpl.inl} (99%) rename cpp/src/db/{MemManager.cpp => MemManager.inl} (97%) diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index 5baa09bd71..610e353358 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -7,8 +7,6 @@ #include "DBImpl.h" #include "DBMetaImpl.h" #include "Env.h" -/* #include "FaissExecutionEngine.h" */ -/* #include "Traits.h" */ #include "Factories.h" namespace zilliz { diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 523922d3d3..6d91a91c7e 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -5,15 +5,15 @@ ******************************************************************************/ #pragma once +#include "DB.h" +#include "MemManager.h" +#include "Types.h" +#include "Traits.h" + #include #include #include #include -#include "DB.h" -#include "MemManager.h" -#include "Types.h" -#include "FaissExecutionEngine.h" -#include "Traits.h" namespace zilliz { namespace vecwise { @@ -95,4 +95,4 @@ private: } // namespace vecwise } // namespace zilliz -#include "DBImpl.cpp" +#include "DBImpl.inl" diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.inl similarity index 99% rename from cpp/src/db/DBImpl.cpp rename to cpp/src/db/DBImpl.inl index 90f0bf5c5d..7adf80d4fe 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.inl @@ -3,8 +3,11 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#ifndef DBIMPL_CPP__ -#define DBIMPL_CPP__ +#pragma once + +#include "DBImpl.h" +#include "DBMetaImpl.h" +#include "Env.h" #include #include @@ -14,10 +17,6 @@ #include #include -#include "DBImpl.h" -#include "DBMetaImpl.h" -#include "Env.h" - namespace zilliz { namespace vecwise { namespace engine { @@ -435,5 +434,3 @@ DBImpl::~DBImpl() { } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index f6cc3a0ecd..69f0fc56a5 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -71,7 +71,7 @@ void DBMetaImpl::GetTableFilePath(TableFileSchema& group_file) { Status DBMetaImpl::NextTableId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; - ss << g.getNextIDNumber(); + ss << g.GetNextIDNumber(); table_id = ss.str(); return Status::OK(); } @@ -79,7 +79,7 @@ Status DBMetaImpl::NextTableId(std::string& table_id) { Status DBMetaImpl::NextFileId(std::string& file_id) { std::stringstream ss; SimpleIDGenerator g; - ss << g.getNextIDNumber(); + ss << g.GetNextIDNumber(); file_id = ss.str(); return Status::OK(); } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 87aae6afbb..191d3dbae5 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -3,6 +3,11 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "Factories.h" +#include "DBImpl.h" +#include "FaissExecutionEngine.h" +#include "Traits.h" + #include #include #include @@ -11,12 +16,6 @@ #include #include -#include "Factories.h" -#include "DBImpl.h" -#include "FaissExecutionEngine.h" -#include "Traits.h" - - namespace zilliz { namespace vecwise { namespace engine { diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 3c9c3a99aa..0e9ab71bb1 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -3,15 +3,15 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// - #pragma once -#include -#include #include "DB.h" #include "DBMetaImpl.h" #include "Options.h" +#include +#include + namespace zilliz { namespace vecwise { namespace engine { diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index 633ee5fe69..d6e71091e6 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -3,30 +3,29 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "IDGenerator.h" + #include #include #include -#include "IDGenerator.h" - - namespace zilliz { namespace vecwise { namespace engine { IDGenerator::~IDGenerator() {} -IDNumber SimpleIDGenerator::getNextIDNumber() { +IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( now.time_since_epoch()).count(); return micros * MAX_IDS_PER_MICRO; } -void SimpleIDGenerator::nextIDNumbers(size_t n, IDNumbers& ids) { +void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { if (n > MAX_IDS_PER_MICRO) { - nextIDNumbers(n-MAX_IDS_PER_MICRO, ids); - nextIDNumbers(MAX_IDS_PER_MICRO, ids); + NextIDNumbers(n-MAX_IDS_PER_MICRO, ids); + NextIDNumbers(MAX_IDS_PER_MICRO, ids); return; } if (n <= 0) { @@ -41,12 +40,11 @@ void SimpleIDGenerator::nextIDNumbers(size_t n, IDNumbers& ids) { for (int pos=0; pos #include "Types.h" +#include +#include + namespace zilliz { namespace vecwise { namespace engine { class IDGenerator { public: - virtual IDNumber getNextIDNumber() = 0; - virtual void getNextIDNumbers(size_t n, IDNumbers& ids) = 0; + virtual IDNumber GetNextIDNumber() = 0; + virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; virtual ~IDGenerator(); @@ -24,11 +26,11 @@ public: class SimpleIDGenerator : public IDGenerator { public: - virtual IDNumber getNextIDNumber() override; - virtual void getNextIDNumbers(size_t n, IDNumbers& ids) override; + virtual IDNumber GetNextIDNumber() override; + virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override; private: - void nextIDNumbers(size_t n, IDNumbers& ids); + void NextIDNumbers(size_t n, IDNumbers& ids); const size_t MAX_IDS_PER_MICRO = 1000; }; // SimpleIDGenerator diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index b6e4b86c89..f45f8c34b2 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -5,15 +5,15 @@ ******************************************************************************/ #pragma once +#include "IDGenerator.h" +#include "Status.h" +#include "Meta.h" + #include #include #include #include #include -#include "IDGenerator.h" -#include "Status.h" -#include "Meta.h" - namespace zilliz { namespace vecwise { @@ -96,4 +96,4 @@ private: } // namespace engine } // namespace vecwise } // namespace zilliz -#include "MemManager.cpp" +#include "MemManager.inl" diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.inl similarity index 97% rename from cpp/src/db/MemManager.cpp rename to cpp/src/db/MemManager.inl index 03ed3de1f3..82efd63d24 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.inl @@ -3,18 +3,16 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#ifndef MEMMANGE_CPP__ -#define MEMMANGE_CPP__ - -#include -#include -#include -#include +#pragma once #include "MemManager.h" #include "Meta.h" #include "MetaConsts.h" +#include +#include +#include +#include namespace zilliz { namespace vecwise { @@ -32,7 +30,7 @@ MemVectors::MemVectors(const std::shared_ptr& meta_ptr, template void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - _pIdGenerator->getNextIDNumbers(n_, vector_ids_); + _pIdGenerator->GetNextIDNumbers(n_, vector_ids_); pEE_->AddWithIds(n_, vectors_, vector_ids_.data()); } @@ -148,5 +146,3 @@ Status MemManager::serialize(std::vector& table_ids) { } // namespace engine } // namespace vecwise } // namespace zilliz - -#endif diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index e459bab4bb..9b40152587 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -3,9 +3,9 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ +#include "Utils.h" #include -#include "Utils.h" namespace zilliz { namespace vecwise { From c8bcf53d81173b097bc8273667434d09d4683a92 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 20:41:22 +0800 Subject: [PATCH 26/31] refactor(db): refactor for db impl Former-commit-id: f7488d5189ed1c34d9b3e8e3da4eaa71766f9c11 --- cpp/src/db/DBImpl.h | 44 +++++++------- cpp/src/db/DBImpl.inl | 138 ++++++++++++++++++++---------------------- 2 files changed, 89 insertions(+), 93 deletions(-) diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 6d91a91c7e..6e596ff3a2 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -35,10 +35,10 @@ public: virtual Status add_group(meta::TableSchema& table_schema) override; virtual Status get_group(meta::TableSchema& table_schema) override; - virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override; + virtual Status has_group(const std::string& table_id, bool& has_or_not) override; - virtual Status add_vectors(const std::string& table_id_, - size_t n, const float* vectors, IDNumbers& vector_ids_) override; + virtual Status add_vectors(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids) override; virtual Status search(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) override; @@ -54,30 +54,30 @@ public: private: - void background_build_index(); - Status build_index(const meta::TableFileSchema&); - Status try_build_index(); - Status merge_files(const std::string& table_id, + void BackgroundBuildIndex(); + Status BuildIndex(const meta::TableFileSchema&); + Status TryBuildIndex(); + Status MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files); - Status background_merge_files(const std::string& table_id); + Status BackgroundMergeFiles(const std::string& table_id); - void try_schedule_compaction(); - void start_timer_task(int interval_); - void background_timer_task(int interval_); + void TrySchedule(); + void StartTimerTasks(int interval); + void BackgroundTimerTask(int interval); static void BGWork(void* db); - void background_call(); - void background_compaction(); + void BackgroundCall(); + void BackgroundCompaction(); - Env* const _env; - const Options _options; + Env* const env_; + const Options options_; - std::mutex _mutex; - std::condition_variable _bg_work_finish_signal; - bool _bg_compaction_scheduled; - Status _bg_error; - std::atomic _shutting_down; + std::mutex mutex_; + std::condition_variable bg_work_finish_signal_; + bool bg_compaction_scheduled_; + Status bg_error_; + std::atomic shutting_down_; std::mutex build_index_mutex_; bool bg_build_index_started_; @@ -85,8 +85,8 @@ private: std::thread bg_timer_thread_; - MetaPtr _pMeta; - MemManagerPtr _pMemMgr; + MetaPtr pMeta_; + MemManagerPtr pMemMgr_; }; // DBImpl diff --git a/cpp/src/db/DBImpl.inl b/cpp/src/db/DBImpl.inl index 7adf80d4fe..025d7c7b19 100644 --- a/cpp/src/db/DBImpl.inl +++ b/cpp/src/db/DBImpl.inl @@ -23,35 +23,35 @@ namespace engine { template DBImpl::DBImpl(const Options& options) - : _env(options.env), - _options(options), - _bg_compaction_scheduled(false), - _shutting_down(false), + : env_(options.env), + options_(options), + bg_compaction_scheduled_(false), + shutting_down_(false), bg_build_index_started_(false), - _pMeta(new meta::DBMetaImpl(_options.meta)), - _pMemMgr(new MemManager(_pMeta, _options)) { - start_timer_task(_options.memory_sync_interval); + pMeta_(new meta::DBMetaImpl(options_.meta)), + pMemMgr_(new MemManager(pMeta_, options_)) { + StartTimerTasks(options_.memory_sync_interval); } template Status DBImpl::add_group(meta::TableSchema& table_schema) { - return _pMeta->CreateTable(table_schema); + return pMeta_->CreateTable(table_schema); } template Status DBImpl::get_group(meta::TableSchema& table_schema) { - return _pMeta->DescribeTable(table_schema); + return pMeta_->DescribeTable(table_schema); } template Status DBImpl::has_group(const std::string& table_id, bool& has_or_not) { - return _pMeta->HasTable(table_id, has_or_not); + return pMeta_->HasTable(table_id, has_or_not); } template Status DBImpl::add_vectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { - Status status = _pMemMgr->add_vectors(table_id_, n, vectors, vector_ids_); + Status status = pMemMgr_->add_vectors(table_id_, n, vectors, vector_ids_); if (!status.ok()) { return status; } @@ -69,7 +69,7 @@ Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; - auto status = _pMeta->FilesToSearch(table_id, dates, files); + auto status = pMeta_->FilesToSearch(table_id, dates, files); if (!status.ok()) { return status; } LOG(DEBUG) << "Search DateT Size=" << files.size(); @@ -195,59 +195,59 @@ Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, } template -void DBImpl::start_timer_task(int interval_) { - bg_timer_thread_ = std::thread(&DBImpl::background_timer_task, this, interval_); +void DBImpl::StartTimerTasks(int interval) { + bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this, interval); } template -void DBImpl::background_timer_task(int interval_) { +void DBImpl::BackgroundTimerTask(int interval) { Status status; while (true) { - if (!_bg_error.ok()) break; - if (_shutting_down.load(std::memory_order_acquire)) break; + if (!bg_error_.ok()) break; + if (shutting_down_.load(std::memory_order_acquire)) break; - std::this_thread::sleep_for(std::chrono::seconds(interval_)); + std::this_thread::sleep_for(std::chrono::seconds(interval)); - try_schedule_compaction(); + TrySchedule(); } } template -void DBImpl::try_schedule_compaction() { - if (_bg_compaction_scheduled) return; - if (!_bg_error.ok()) return; +void DBImpl::TrySchedule() { + if (bg_compaction_scheduled_) return; + if (!bg_error_.ok()) return; - _bg_compaction_scheduled = true; - _env->Schedule(&DBImpl::BGWork, this); + bg_compaction_scheduled_ = true; + env_->Schedule(&DBImpl::BGWork, this); } template void DBImpl::BGWork(void* db_) { - reinterpret_cast(db_)->background_call(); + reinterpret_cast(db_)->BackgroundCall(); } template -void DBImpl::background_call() { - std::lock_guard lock(_mutex); - assert(_bg_compaction_scheduled); +void DBImpl::BackgroundCall() { + std::lock_guard lock(mutex_); + assert(bg_compaction_scheduled_); - if (!_bg_error.ok() || _shutting_down.load(std::memory_order_acquire)) + if (!bg_error_.ok() || shutting_down_.load(std::memory_order_acquire)) return ; - background_compaction(); + BackgroundCompaction(); - _bg_compaction_scheduled = false; - _bg_work_finish_signal.notify_all(); + bg_compaction_scheduled_ = false; + bg_work_finish_signal_.notify_all(); } template -Status DBImpl::merge_files(const std::string& table_id, const meta::DateT& date, +Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { meta::TableFileSchema table_file; table_file.table_id = table_id; table_file.date = date; - Status status = _pMeta->CreateTableFile(table_file); + Status status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { LOG(INFO) << status.ToString() << std::endl; @@ -267,19 +267,19 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat LOG(DEBUG) << "Merging file " << file_schema.file_id; index_size = index.Size(); - if (index_size >= _options.index_trigger_size) break; + if (index_size >= options_.index_trigger_size) break; } index.Serialize(); - if (index_size >= _options.index_trigger_size) { + if (index_size >= options_.index_trigger_size) { table_file.file_type = meta::TableFileSchema::TO_INDEX; } else { table_file.file_type = meta::TableFileSchema::RAW; } table_file.size = index_size; updated.push_back(table_file); - status = _pMeta->UpdateTableFiles(updated); + status = pMeta_->UpdateTableFiles(updated); LOG(DEBUG) << "New merged file " << table_file.file_id << " of size=" << index.PhysicalSize()/(1024*1024) << " M"; @@ -289,43 +289,39 @@ Status DBImpl::merge_files(const std::string& table_id, const meta::Dat } template -Status DBImpl::background_merge_files(const std::string& table_id) { +Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; - auto status = _pMeta->FilesToMerge(table_id, raw_files); + auto status = pMeta_->FilesToMerge(table_id, raw_files); if (!status.ok()) { return status; } - /* if (raw_files.size() == 0) { */ - /* return Status::OK(); */ - /* } */ - bool has_merge = false; for (auto& kv : raw_files) { auto files = kv.second; - if (files.size() <= _options.merge_trigger_number) { + if (files.size() <= options_.merge_trigger_number) { continue; } has_merge = true; - merge_files(table_id, kv.first, kv.second); + MergeFiles(table_id, kv.first, kv.second); } - _pMeta->Archive(); + pMeta_->Archive(); - try_build_index(); + TryBuildIndex(); - _pMeta->CleanUpFilesWithTTL(1); + pMeta_->CleanUpFilesWithTTL(1); return Status::OK(); } template -Status DBImpl::build_index(const meta::TableFileSchema& file) { +Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; table_file.table_id = file.table_id; table_file.date = file.date; - Status status = _pMeta->CreateTableFile(table_file); + Status status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { return status; } @@ -342,30 +338,30 @@ Status DBImpl::build_index(const meta::TableFileSchema& file) { to_remove.file_type = meta::TableFileSchema::TO_DELETE; meta::TableFilesSchema update_files = {to_remove, table_file}; - _pMeta->UpdateTableFiles(update_files); + pMeta_->UpdateTableFiles(update_files); LOG(DEBUG) << "New index file " << table_file.file_id << " of size " << index->PhysicalSize()/(1024*1024) << " M" << " from file " << to_remove.file_id; index->Cache(); - _pMeta->Archive(); + pMeta_->Archive(); return Status::OK(); } template -void DBImpl::background_build_index() { +void DBImpl::BackgroundBuildIndex() { std::lock_guard lock(build_index_mutex_); assert(bg_build_index_started_); meta::TableFilesSchema to_index_files; - _pMeta->FilesToIndex(to_index_files); + pMeta_->FilesToIndex(to_index_files); Status status; for (auto& file : to_index_files) { /* LOG(DEBUG) << "Buiding index for " << file.location; */ - status = build_index(file); + status = BuildIndex(file); if (!status.ok()) { - _bg_error = status; + bg_error_ = status; return; } } @@ -376,25 +372,25 @@ void DBImpl::background_build_index() { } template -Status DBImpl::try_build_index() { +Status DBImpl::TryBuildIndex() { if (bg_build_index_started_) return Status::OK(); - if (_shutting_down.load(std::memory_order_acquire)) return Status::OK(); + if (shutting_down_.load(std::memory_order_acquire)) return Status::OK(); bg_build_index_started_ = true; - std::thread build_index_task(&DBImpl::background_build_index, this); + std::thread build_index_task(&DBImpl::BackgroundBuildIndex, this); build_index_task.detach(); return Status::OK(); } template -void DBImpl::background_compaction() { +void DBImpl::BackgroundCompaction() { std::vector table_ids; - _pMemMgr->serialize(table_ids); + pMemMgr_->serialize(table_ids); Status status; for (auto table_id : table_ids) { - status = background_merge_files(table_id); + status = BackgroundMergeFiles(table_id); if (!status.ok()) { - _bg_error = status; + bg_error_ = status; return; } } @@ -402,21 +398,21 @@ void DBImpl::background_compaction() { template Status DBImpl::drop_all() { - return _pMeta->DropAll(); + return pMeta_->DropAll(); } template Status DBImpl::size(long& result) { - return _pMeta->Size(result); + return pMeta_->Size(result); } template DBImpl::~DBImpl() { { - std::unique_lock lock(_mutex); - _shutting_down.store(true, std::memory_order_release); - while (_bg_compaction_scheduled) { - _bg_work_finish_signal.wait(lock); + std::unique_lock lock(mutex_); + shutting_down_.store(true, std::memory_order_release); + while (bg_compaction_scheduled_) { + bg_work_finish_signal_.wait(lock); } } { @@ -427,8 +423,8 @@ DBImpl::~DBImpl() { } bg_timer_thread_.join(); std::vector ids; - _pMemMgr->serialize(ids); - _env->Stop(); + pMemMgr_->serialize(ids); + env_->Stop(); } } // namespace engine From 130b7101699fc0fdcd614f9f8b269dd1a6b6de40 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Mon, 27 May 2019 20:43:07 +0800 Subject: [PATCH 27/31] refactor(db): MS-20 Former-commit-id: f34dd2746073c837fa877697e32f1ce3607e35c2 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 489228eca8..1fbd50029a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. ### New Feature - MS-5 - Implement Auto Archive Feature +- MS-20 - Clean Code Part 1 ### Task From c5501202adec84b12abdca84be9ad16594edef05 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Tue, 28 May 2019 13:49:59 +0800 Subject: [PATCH 28/31] refactor(db): whole db api refactor Former-commit-id: 3519e8a464bcd66b6da8ef2c579c08eaf13d17bd --- cpp/src/db/DB.h | 16 ++++---- cpp/src/db/DBImpl.h | 16 ++++---- cpp/src/db/DBImpl.inl | 24 ++++++------ cpp/src/db/MemManager.h | 54 +++++++++++++------------- cpp/src/db/MemManager.inl | 64 +++++++++++++++---------------- cpp/src/server/MegasearchTask.cpp | 44 ++++++++++----------- cpp/unittest/db/db_tests.cpp | 30 +++++++-------- 7 files changed, 124 insertions(+), 124 deletions(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index c6859b35c6..1db4d72356 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -21,22 +21,22 @@ class DB { public: static void Open(const Options& options, DB** dbptr); - virtual Status add_group(meta::TableSchema& table_schema_) = 0; - virtual Status get_group(meta::TableSchema& table_schema_) = 0; - virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0; + virtual Status CreateTable(meta::TableSchema& table_schema_) = 0; + virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0; + virtual Status HasTable(const std::string& table_id_, bool& has_or_not_) = 0; - virtual Status add_vectors(const std::string& table_id_, + virtual Status InsertVectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) = 0; - virtual Status search(const std::string& table_id, size_t k, size_t nq, + virtual Status Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) = 0; - virtual Status search(const std::string& table_id, size_t k, size_t nq, + virtual Status Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; - virtual Status size(long& result) = 0; + virtual Status Size(long& result) = 0; - virtual Status drop_all() = 0; + virtual Status DropAll() = 0; DB() = default; DB(const DB&) = delete; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 6e596ff3a2..0c84af8f0c 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -33,22 +33,22 @@ public: DBImpl(const Options& options); - virtual Status add_group(meta::TableSchema& table_schema) override; - virtual Status get_group(meta::TableSchema& table_schema) override; - virtual Status has_group(const std::string& table_id, bool& has_or_not) override; + virtual Status CreateTable(meta::TableSchema& table_schema) override; + virtual Status DescribeTable(meta::TableSchema& table_schema) override; + virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status add_vectors(const std::string& table_id, + virtual Status InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) override; - virtual Status search(const std::string& table_id, size_t k, size_t nq, + virtual Status Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, QueryResults& results) override; - virtual Status search(const std::string& table_id, size_t k, size_t nq, + virtual Status Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) override; - virtual Status drop_all() override; + virtual Status DropAll() override; - virtual Status size(long& result) override; + virtual Status Size(long& result) override; virtual ~DBImpl(); diff --git a/cpp/src/db/DBImpl.inl b/cpp/src/db/DBImpl.inl index 025d7c7b19..d061a99959 100644 --- a/cpp/src/db/DBImpl.inl +++ b/cpp/src/db/DBImpl.inl @@ -34,38 +34,38 @@ DBImpl::DBImpl(const Options& options) } template -Status DBImpl::add_group(meta::TableSchema& table_schema) { +Status DBImpl::CreateTable(meta::TableSchema& table_schema) { return pMeta_->CreateTable(table_schema); } template -Status DBImpl::get_group(meta::TableSchema& table_schema) { +Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { return pMeta_->DescribeTable(table_schema); } template -Status DBImpl::has_group(const std::string& table_id, bool& has_or_not) { +Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { return pMeta_->HasTable(table_id, has_or_not); } template -Status DBImpl::add_vectors(const std::string& table_id_, +Status DBImpl::InsertVectors(const std::string& table_id_, size_t n, const float* vectors, IDNumbers& vector_ids_) { - Status status = pMemMgr_->add_vectors(table_id_, n, vectors, vector_ids_); + Status status = pMemMgr_->InsertVectors(table_id_, n, vectors, vector_ids_); if (!status.ok()) { return status; } } template -Status DBImpl::search(const std::string &table_id, size_t k, size_t nq, +Status DBImpl::Query(const std::string &table_id, size_t k, size_t nq, const float *vectors, QueryResults &results) { meta::DatesT dates = {meta::Meta::GetDate()}; - return search(table_id, k, nq, vectors, dates, results); + return Query(table_id, k, nq, vectors, dates, results); } template -Status DBImpl::search(const std::string& table_id, size_t k, size_t nq, +Status DBImpl::Query(const std::string& table_id, size_t k, size_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { meta::DatePartionedTableFilesSchema files; @@ -384,7 +384,7 @@ Status DBImpl::TryBuildIndex() { template void DBImpl::BackgroundCompaction() { std::vector table_ids; - pMemMgr_->serialize(table_ids); + pMemMgr_->Serialize(table_ids); Status status; for (auto table_id : table_ids) { @@ -397,12 +397,12 @@ void DBImpl::BackgroundCompaction() { } template -Status DBImpl::drop_all() { +Status DBImpl::DropAll() { return pMeta_->DropAll(); } template -Status DBImpl::size(long& result) { +Status DBImpl::Size(long& result) { return pMeta_->Size(result); } @@ -423,7 +423,7 @@ DBImpl::~DBImpl() { } bg_timer_thread_.join(); std::vector ids; - pMemMgr_->serialize(ids); + pMemMgr_->Serialize(ids); env_->Stop(); } diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index f45f8c34b2..35272f4211 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -26,24 +26,24 @@ namespace meta { template class MemVectors { public: - typedef typename EngineT::Ptr EnginePtr; - typedef typename meta::Meta::Ptr MetaPtr; - typedef std::shared_ptr> Ptr; + using EnginePtr = typename EngineT::Ptr; + using MetaPtr = meta::Meta::Ptr; + using Ptr = std::shared_ptr>; explicit MemVectors(const std::shared_ptr&, const meta::TableFileSchema&, const Options&); - void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); + void Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); - size_t total() const; + size_t Total() const; - size_t approximate_size() const; + size_t ApproximateSize() const; - Status serialize(std::string& table_id); + Status Serialize(std::string& table_id); ~MemVectors(); - const std::string& location() const { return schema_.location; } + const std::string& Location() const { return schema_.location; } private: MemVectors() = delete; @@ -53,7 +53,7 @@ private: MetaPtr pMeta_; Options options_; meta::TableFileSchema schema_; - IDGenerator* _pIdGenerator; + IDGenerator* pIdGenerator_; EnginePtr pEE_; }; // MemVectors @@ -63,32 +63,32 @@ private: template class MemManager { public: - typedef typename meta::Meta::Ptr MetaPtr; - typedef typename MemVectors::Ptr MemVectorsPtr; - typedef std::shared_ptr> Ptr; + using MetaPtr = meta::Meta::Ptr; + using MemVectorsPtr = typename MemVectors::Ptr; + using Ptr = std::shared_ptr>; - MemManager(const std::shared_ptr& meta_, const Options& options) - : _pMeta(meta_), options_(options) {} + MemManager(const std::shared_ptr& meta, const Options& options) + : pMeta_(meta), options_(options) {} - MemVectorsPtr get_mem_by_group(const std::string& table_id_); + MemVectorsPtr GetMemByTable(const std::string& table_id); - Status add_vectors(const std::string& table_id_, - size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status InsertVectors(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids); - Status serialize(std::vector& table_ids); + Status Serialize(std::vector& table_ids); private: - Status add_vectors_no_lock(const std::string& table_id_, - size_t n_, const float* vectors_, IDNumbers& vector_ids_); - Status mark_memory_as_immutable(); + Status InsertVectorsNoLock(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids); + Status ToImmutable(); - typedef std::map MemMap; - typedef std::vector ImmMemPool; - MemMap _memMap; - ImmMemPool _immMems; - MetaPtr _pMeta; + using MemMap = std::map; + using ImmMemPool = std::vector; + MemMap memMap_; + ImmMemPool immMems_; + MetaPtr pMeta_; Options options_; - std::mutex _mutex; + std::mutex mutex_; std::mutex serialization_mtx_; }; // MemManager diff --git a/cpp/src/db/MemManager.inl b/cpp/src/db/MemManager.inl index 82efd63d24..35e7c70ada 100644 --- a/cpp/src/db/MemManager.inl +++ b/cpp/src/db/MemManager.inl @@ -24,30 +24,30 @@ MemVectors::MemVectors(const std::shared_ptr& meta_ptr, : pMeta_(meta_ptr), options_(options), schema_(schema), - _pIdGenerator(new SimpleIDGenerator()), + pIdGenerator_(new SimpleIDGenerator()), pEE_(new EngineT(schema_.dimension, schema_.location)) { } template -void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - _pIdGenerator->GetNextIDNumbers(n_, vector_ids_); +void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { + pIdGenerator_->GetNextIDNumbers(n_, vector_ids_); pEE_->AddWithIds(n_, vectors_, vector_ids_.data()); } template -size_t MemVectors::total() const { +size_t MemVectors::Total() const { return pEE_->Count(); } template -size_t MemVectors::approximate_size() const { +size_t MemVectors::ApproximateSize() const { return pEE_->Size(); } template -Status MemVectors::serialize(std::string& table_id) { +Status MemVectors::Serialize(std::string& table_id) { table_id = schema_.table_id; - auto size = approximate_size(); + auto size = ApproximateSize(); pEE_->Serialize(); schema_.size = size; schema_.file_type = (size >= options_.index_trigger_size) ? @@ -65,9 +65,9 @@ Status MemVectors::serialize(std::string& table_id) { template MemVectors::~MemVectors() { - if (_pIdGenerator != nullptr) { - delete _pIdGenerator; - _pIdGenerator = nullptr; + if (pIdGenerator_ != nullptr) { + delete pIdGenerator_; + pIdGenerator_ = nullptr; } } @@ -76,69 +76,69 @@ MemVectors::~MemVectors() { */ template -typename MemManager::MemVectorsPtr MemManager::get_mem_by_group( +typename MemManager::MemVectorsPtr MemManager::GetMemByTable( const std::string& table_id) { - auto memIt = _memMap.find(table_id); - if (memIt != _memMap.end()) { + auto memIt = memMap_.find(table_id); + if (memIt != memMap_.end()) { return memIt->second; } meta::TableFileSchema table_file; table_file.table_id = table_id; - auto status = _pMeta->CreateTableFile(table_file); + auto status = pMeta_->CreateTableFile(table_file); if (!status.ok()) { return nullptr; } - _memMap[table_id] = MemVectorsPtr(new MemVectors(_pMeta, table_file, options_)); - return _memMap[table_id]; + memMap_[table_id] = MemVectorsPtr(new MemVectors(pMeta_, table_file, options_)); + return memMap_[table_id]; } template -Status MemManager::add_vectors(const std::string& table_id_, +Status MemManager::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - std::unique_lock lock(_mutex); - return add_vectors_no_lock(table_id_, n_, vectors_, vector_ids_); + std::unique_lock lock(mutex_); + return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } template -Status MemManager::add_vectors_no_lock(const std::string& table_id, +Status MemManager::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { - MemVectorsPtr mem = get_mem_by_group(table_id); + MemVectorsPtr mem = GetMemByTable(table_id); if (mem == nullptr) { return Status::NotFound("Group " + table_id + " not found!"); } - mem->add(n, vectors, vector_ids); + mem->Add(n, vectors, vector_ids); return Status::OK(); } template -Status MemManager::mark_memory_as_immutable() { - std::unique_lock lock(_mutex); - for (auto& kv: _memMap) { - _immMems.push_back(kv.second); +Status MemManager::ToImmutable() { + std::unique_lock lock(mutex_); + for (auto& kv: memMap_) { + immMems_.push_back(kv.second); } - _memMap.clear(); + memMap_.clear(); return Status::OK(); } template -Status MemManager::serialize(std::vector& table_ids) { - mark_memory_as_immutable(); +Status MemManager::Serialize(std::vector& table_ids) { + ToImmutable(); std::unique_lock lock(serialization_mtx_); std::string table_id; table_ids.clear(); - for (auto& mem : _immMems) { - mem->serialize(table_id); + for (auto& mem : immMems_) { + mem->Serialize(table_id); table_ids.push_back(table_id); } - _immMems.clear(); + immMems_.clear(); return Status::OK(); } diff --git a/cpp/src/server/MegasearchTask.cpp b/cpp/src/server/MegasearchTask.cpp index 884557ffa4..6f01b38ce1 100644 --- a/cpp/src/server/MegasearchTask.cpp +++ b/cpp/src/server/MegasearchTask.cpp @@ -78,17 +78,17 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) { ServerError CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); - + try { if(schema_.vector_column_array.empty()) { return SERVER_INVALID_ARGUMENT; } IVecIdMapper::GetInstance()->AddGroup(schema_.table_name); - engine::meta::GroupSchema group_info; - group_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension; - group_info.group_id = schema_.table_name; - engine::Status stat = DB()->add_group(group_info); + engine::meta::TableSchema table_schema; + table_schema.dimension = (uint16_t)schema_.vector_column_array[0].dimension; + table_schema.table_id = schema_.table_name; + engine::Status stat = DB()->CreateTable(table_schema); if(!stat.ok()) {//could exist error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; @@ -123,9 +123,9 @@ ServerError DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { - engine::meta::GroupSchema group_info; - group_info.group_id = table_name_; - engine::Status stat = DB()->get_group(group_info); + engine::meta::TableSchema table_schema; + table_schema.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_schema); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -154,8 +154,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name) } -BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) { - return std::shared_ptr(new DeleteTableTask(group_id)); +BaseTaskPtr DeleteTableTask::Create(const std::string& table_id) { + return std::shared_ptr(new DeleteTableTask(table_id)); } ServerError DeleteTableTask::OnExecute() { @@ -195,9 +195,9 @@ ServerError AddVectorTask::OnExecute() { return SERVER_SUCCESS; } - engine::meta::GroupSchema group_info; - group_info.group_id = table_name_; - engine::Status stat = DB()->get_group(group_info); + engine::meta::TableSchema table_schema; + table_schema.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_schema); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -208,7 +208,7 @@ ServerError AddVectorTask::OnExecute() { rc.Record("get group info"); uint64_t vec_count = (uint64_t)record_array_.size(); - uint64_t group_dim = group_info.dimension; + uint64_t group_dim = table_schema.dimension; std::vector vec_f; vec_f.resize(vec_count*group_dim);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { @@ -236,7 +236,7 @@ ServerError AddVectorTask::OnExecute() { rc.Record("prepare vectors data"); - stat = DB()->add_vectors(table_name_, vec_count, vec_f.data(), record_ids_); + stat = DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; @@ -293,9 +293,9 @@ ServerError SearchVectorTask::OnExecute() { return error_code_; } - engine::meta::GroupSchema group_info; - group_info.group_id = table_name_; - engine::Status stat = DB()->get_group(group_info); + engine::meta::TableSchema table_schema; + table_schema.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_schema); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -305,7 +305,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector vec_f; uint64_t record_count = (uint64_t)record_array_.size(); - vec_f.resize(record_count*group_info.dimension); + vec_f.resize(record_count*table_schema.dimension); for(uint64_t i = 0; i < record_array_.size(); i++) { const auto& record = record_array_[i]; @@ -317,9 +317,9 @@ ServerError SearchVectorTask::OnExecute() { } uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value? - if (vec_dim != group_info.dimension) { + if (vec_dim != table_schema.dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << group_info.dimension; + << " vs. group dimension:" << table_schema.dimension; error_code_ = SERVER_INVALID_VECTOR_DIMENSION; error_msg_ = "Engine failed: " + stat.ToString(); return error_code_; @@ -335,7 +335,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector dates; engine::QueryResults results; - stat = DB()->search(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results); + stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results); if(!stat.ok()) { SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a5d5bcf1fc..4d2bfb3643 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -69,18 +69,18 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.table_id = group_name; - engine::Status stat = db_->add_group(group_info); + engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; group_info_get.table_id = group_name; - stat = db_->get_group(group_info_get); + stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; - db_->size(size); + db_->Size(size); int d = 256; int nb = 20; float *xb = new float[d * nb]; @@ -92,13 +92,13 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { int loop = 100000; for (auto i=0; iadd_vectors(group_name, nb, xb, vector_ids); + db_->InsertVectors(group_name, nb, xb, vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } std::this_thread::sleep_for(std::chrono::seconds(1)); - db_->size(size); + db_->Size(size); LOG(DEBUG) << "size=" << size; ASSERT_TRUE(size < 1 * engine::meta::G); @@ -114,11 +114,11 @@ TEST_F(DBTest, DB_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.table_id = group_name; - engine::Status stat = db_->add_group(group_info); + engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; group_info_get.table_id = group_name; - stat = db_->get_group(group_info_get); + stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -152,11 +152,11 @@ TEST_F(DBTest, DB_TEST) { for (auto j=0; j<10; ++j) { ss.str(""); - db_->size(count); + db_->Size(count); prev_count = count; START_TIMER; - stat = db_->search(group_name, k, qb, qxb, results); + stat = db_->Query(group_name, k, qb, qxb, results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -179,10 +179,10 @@ TEST_F(DBTest, DB_TEST) { for (auto i=0; iadd_vectors(group_name, qb, qxb, target_ids); + db_->InsertVectors(group_name, qb, qxb, target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->add_vectors(group_name, nb, xb, vector_ids); + db_->InsertVectors(group_name, nb, xb, vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -200,11 +200,11 @@ TEST_F(DBTest, SEARCH_TEST) { engine::meta::TableSchema group_info; group_info.dimension = group_dim; group_info.table_id = group_name; - engine::Status stat = db_->add_group(group_info); + engine::Status stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; group_info_get.table_id = group_name; - stat = db_->get_group(group_info_get); + stat = db_->DescribeTable(group_info_get); ASSERT_STATS(stat); ASSERT_EQ(group_info_get.dimension, group_dim); @@ -238,7 +238,7 @@ TEST_F(DBTest, SEARCH_TEST) { // insert data const int batch_size = 100; for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->add_vectors(group_name, batch_size, xb.data()+batch_size*j*group_dim, ids); + stat = db_->InsertVectors(group_name, batch_size, xb.data()+batch_size*j*group_dim, ids); if (j == 200){ sleep(1);} ASSERT_STATS(stat); } @@ -246,7 +246,7 @@ TEST_F(DBTest, SEARCH_TEST) { sleep(2); // wait until build index finish engine::QueryResults results; - stat = db_->search(group_name, k, nq, xq.data(), results); + stat = db_->Query(group_name, k, nq, xq.data(), results); ASSERT_STATS(stat); // TODO(linxj): add groundTruth assert From 9b5791b434a867dede831ceab37bbeba3cee2181 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Tue, 28 May 2019 13:59:16 +0800 Subject: [PATCH 29/31] refactor(db): small change Former-commit-id: 057c7a7b4b9189895d7aac3e253387fe91daee0e --- cpp/src/db/DB.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 1db4d72356..0f1db62cfb 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -5,12 +5,13 @@ ******************************************************************************/ #pragma once -#include #include "Options.h" #include "Meta.h" #include "Status.h" #include "Types.h" +#include + namespace zilliz { namespace vecwise { namespace engine { From 120a52c184e43f4abdb79135111a8d9a81d6d2e4 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Tue, 28 May 2019 14:01:54 +0800 Subject: [PATCH 30/31] refactor(db): replace typedef with using Former-commit-id: 62caeac6fb8a05cabe0622abe73d7b5b548f8472 --- cpp/src/db/DBImpl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 0c84af8f0c..7384e53a34 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -28,8 +28,8 @@ namespace meta { template class DBImpl : public DB { public: - typedef typename meta::Meta::Ptr MetaPtr; - typedef typename MemManager::Ptr MemManagerPtr; + using MetaPtr = meta::Meta::Ptr; + using MemManagerPtr = typename MemManager::Ptr; DBImpl(const Options& options); From 13ef8219aa53b1bc4d8529a13a55f478eb041019 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Tue, 28 May 2019 14:03:09 +0800 Subject: [PATCH 31/31] refactor(db): update change log Former-commit-id: 95c0f52c4f6b1b6723e72e71baed6bc640b7935f --- cpp/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1fbd50029a..a5db7a5019 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,10 +11,10 @@ Please mark all change in change log and use the ticket from JIRA. ### New Feature - MS-5 - Implement Auto Archive Feature -- MS-20 - Clean Code Part 1 ### Task - MS-1 - Add CHANGELOG.md - MS-4 - Refactor the vecwise_engine code structure - MS-6 - Implement SDK interface part 1 +- MS-20 - Clean Code Part 1