admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-02 17:05:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(db): impl build index

Browse Source 
Former-commit-id: 3fd41e5fa861957e108a4d077a152dc0c30f2691
This commit is contained in:
peng.xu 2019-07-09 22:08:21 +08:00
parent 710ae19ded
commit c6d38f0af2
7 changed files with 50 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cpp/src/db/DBImpl.cpp
View File

@ -408,10 +408,10 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->CleanUpFilesWithTTL(ttl);
}
void DBImpl::StartBuildIndexTask() {
void DBImpl::StartBuildIndexTask(bool force) {
static uint64_t index_clock_tick = 0;
index_clock_tick++;
if(index_clock_tick%INDEX_ACTION_INTERVAL != 0) {
if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) {
return;
}
@ -431,8 +431,20 @@ void DBImpl::StartBuildIndexTask() {
}
Status DBImpl::BuildIndex(const std::string& table_id) {
meta_ptr_->UpdateTableFilesToIndex(table_id);
return BuildIndexByTable(table_id);
bool has = false;
meta_ptr_->HasNonIndexFiles(table_id, has);
int times = 1;
while (has) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
meta_ptr_->UpdateTableFilesToIndex(table_id);
StartBuildIndexTask(true);
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
meta_ptr_->HasNonIndexFiles(table_id, has);
times++;
}
return Status::OK();
/* return BuildIndexByTable(table_id); */
}
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {

2
cpp/src/db/DBImpl.h
View File

@ -109,7 +109,7 @@ class DBImpl : public DB {
Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask();
void StartBuildIndexTask(bool force=false);
void BackgroundBuildIndex();
Status

22
cpp/src/db/DBMetaImpl.cpp
View File

@ -316,6 +316,28 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
return Status::OK();
}
Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
has = false;
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_),
where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
or
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX)
and c(&TableFileSchema::table_id_) == table_id
));
if (selected.size() >= 1) {
has = true;
} else {
has = false;
}
} catch (std::exception &e) {
return HandleException("Encounter exception when check non index files", e);
}
return Status::OK();
}
Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
has_or_not = false;

2
cpp/src/db/DBMetaImpl.h
View File

@ -35,6 +35,8 @@ public:
const std::vector<size_t>& ids,
TableFilesSchema& table_files) override;
virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
virtual Status UpdateTableFile(TableFileSchema& file_schema) override;

2
cpp/src/db/Meta.h
View File

@ -58,6 +58,8 @@ public:
virtual Status FilesToIndex(TableFilesSchema&) = 0;
virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) = 0;
virtual Status CleanUp() = 0;
virtual Status CleanUpFilesWithTTL(uint16_t) = 0;

5
cpp/src/db/MySQLMetaImpl.cpp
View File

@ -436,6 +436,11 @@ namespace meta {
return Status::OK();
}
Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
// TODO
return Status::OK();
}
Status MySQLMetaImpl::DeleteTable(const std::string& table_id) {
// std::lock_guard<std::recursive_mutex> lock(mysql_mutex);

2
cpp/src/db/MySQLMetaImpl.h
View File

@ -40,6 +40,8 @@ namespace meta {
const std::vector<size_t>& ids,
TableFilesSchema& table_files) override;
virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;