admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

MS-427 Describe index error after drop index

Browse Source 
Former-commit-id: c72a434f9f5502c1aee209097bd84fa8ad709fe3
This commit is contained in:
starlord 2019-08-28 19:45:15 +08:00
parent 4be665ee95
commit cf815ea3a2
9 changed files with 61 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash
- MS-417 - YAML sequence load disable cause scheduler startup failed
- MS-413 - Create index failed and server exited
- MS-427 - Describe index error after drop index
## Improvement
- MS-327 - Clean code for milvus

1
cpp/src/db/DB.h
View File

@ -46,7 +46,6 @@ public:
virtual Status Size(uint64_t& result) = 0;
virtual Status BuildIndex(const std::string& table_id) = 0;
virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status DropIndex(const std::string& table_id) = 0;

26
cpp/src/db/DBImpl.cpp
View File

@ -424,7 +424,14 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
}
//step 4: update table files state
table_file.file_type_ = meta::TableFileSchema::RAW;
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
table_file.file_type_ = meta::TableFileSchema::RAW;
}
table_file.file_size_ = index->PhysicalSize();
table_file.row_count_ = index->Count();
updated.push_back(table_file);
@ -516,22 +523,6 @@ void DBImpl::StartBuildIndexTask(bool force) {
}
}
Status DBImpl::BuildIndex(const std::string& table_id) {
bool has = false;
meta_ptr_->HasNonIndexFiles(table_id, has);
int times = 1;
while (has) {
ENGINE_LOG_DEBUG << "Non index files detected in " << table_id << "! Will build index " << times;
meta_ptr_->UpdateTableFilesToIndex(table_id);
/* StartBuildIndexTask(true); */
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
meta_ptr_->HasNonIndexFiles(table_id, has);
times++;
}
return Status::OK();
}
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
@ -558,7 +549,6 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
}
//step 3: update index info
status = meta_ptr_->UpdateTableIndexParam(table_id, index);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info";

2
cpp/src/db/DBImpl.h
View File

@ -82,8 +82,6 @@ class DBImpl : public DB {
Status Size(uint64_t &result) override;
Status BuildIndex(const std::string& table_id) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override;

10
cpp/src/db/insert/MemTableFile.cpp
View File

@ -87,8 +87,14 @@ Status MemTableFile::Serialize() {
table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
table_file_schema_.row_count_ = execution_engine_->Count();
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
table_file_schema_.file_type_ = meta::TableFileSchema::RAW;
}
auto status = meta_->UpdateTableFile(table_file_schema_);

2
cpp/src/db/meta/MetaTypes.h
View File

@ -19,8 +19,8 @@ namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_USERID = 1;

17
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -536,6 +536,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
Query dropTableIndexQuery = connectionPtr->query();
//soft delete index files
dropTableIndexQuery << "UPDATE TableFiles " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," <<
"updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
@ -550,6 +551,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
dropTableIndexQuery.error());
}
//set all backup file to raw
dropTableIndexQuery << "UPDATE TableFiles " <<
"SET file_type = " << std::to_string(TableFileSchema::RAW) << "," <<
"updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
@ -564,6 +566,21 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
dropTableIndexQuery.error());
}
//set table index type to raw
dropTableIndexQuery << "UPDATE Tables " <<
"SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," <<
"nlist = " << std::to_string(DEFAULT_NLIST) << " " <<
"metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " <<
"WHERE table_id = " << quote << table_id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str();
if (!dropTableIndexQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX";
return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX",
dropTableIndexQuery.error());
}
} //Scoped Connection
} catch (const BadQuery &er) {

17
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -461,6 +461,17 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) {
c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP
));
//set table index type to raw
ConnectorPtr->update_all(
set(
c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE,
c(&TableSchema::nlist_) = DEFAULT_NLIST,
c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE
),
where(
c(&TableSchema::table_id_) == table_id
));
} catch (std::exception &e) {
return HandleException("Encounter exception when delete table index files", e);
}
@ -798,11 +809,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
}
files[table_file.date_].push_back(table_file);
}
if(files.empty()) {
std::cout << "ERROR" << std::endl;
}
} catch (std::exception &e) {
return HandleException("Encounter exception when iterate index files", e);
}
return Status::OK();
}

14
cpp/unittest/db/db_tests.cpp
View File

@ -34,7 +34,6 @@ namespace {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
return table_info;
}
@ -263,7 +262,9 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_STATS(stat);
}
db_->BuildIndex(TABLE_NAME); // wait until build index finish
engine::TableIndex index;
index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
engine::QueryResults results;
@ -273,7 +274,7 @@ TEST_F(DBTest, SEARCH_TEST) {
{//search by specify index file
engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"4", "5", "6"};
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);
ASSERT_STATS(stat);
@ -305,7 +306,12 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), target_ids);
ASSERT_EQ(target_ids.size(), nb);
}
db_->BuildIndex(TABLE_NAME);
sleep(2);
engine::TableIndex index;
index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);