admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge 0.3.1

Browse Source 
Former-commit-id: 6fa5ce5365752369865360d715659c56fc367f89
This commit is contained in:
starlord 2019-08-22 09:21:15 +08:00
parent 223cf1f547
commit 45a777b4d5
11 changed files with 158 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -61,6 +61,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-257 - Update bzip2 download url
- MS-288 - Update compile scripts
- MS-330 - Stability test failed caused by server core dumped
- MS-347 - Build index hangs again
## Improvement
- MS-156 - Add unittest for merge result functions

22
cpp/src/db/Utils.cpp
View File

@ -85,16 +85,20 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK();
}
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id) {
std::string db_path = options.path;
std::string table_path = db_path + TABLES_FOLDER + table_id;
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
std::vector<std::string> paths = options.slave_paths;
paths.push_back(options.path);
for(auto& path : options.slave_paths) {
table_path = path + TABLES_FOLDER + table_id;
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
for(auto& path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
if(force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if(boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
}
}
return Status::OK();

2
cpp/src/db/Utils.h
View File

@ -19,7 +19,7 @@ namespace utils {
long GetMicroSecTimeStamp();
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id);
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);

47
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1652,15 +1652,14 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
}
Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
std::set<std::string> table_ids;
//remove to_delete files
try {
MetricCollector metric;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
@ -1700,6 +1699,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_;
idsToDelete.emplace_back(std::to_string(table_file.id_));
table_ids.insert(table_file.table_id_);
}
if (!idsToDelete.empty()) {
@ -1734,12 +1735,11 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
}
//remove to_delete tables
try {
MetricCollector metric;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
@ -1765,7 +1765,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
std::string table_id;
resRow["table_id"].to_string(table_id);
utils::DeleteTablePath(options_, table_id);
utils::DeleteTablePath(options_, table_id, false);//only delete empty folder
idsToDeleteSS << "id = " << std::to_string(id) << " OR ";
}
@ -1794,6 +1794,39 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
}
//remove deleted table folder
//don't remove table folder until all its files has been deleted
try {
MetricCollector metric;
{
ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
if (connectionPtr == nullptr) {
return Status::Error("Failed to connect to database server");
}
for(auto& table_id : table_ids) {
Query cleanUpFilesWithTTLQuery = connectionPtr->query();
cleanUpFilesWithTTLQuery << "SELECT file_id " <<
"FROM TableFiles " <<
"WHERE table_id = " << table_id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
if (res.empty()) {
utils::DeleteTablePath(options_, table_id);
}
}
}
} catch (const Exception &er) {
// Catch-all for any other MySQL++ exceptions
ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL" << ": " << er.what();
return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", er.what());
}
return Status::OK();
}

24
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -1069,6 +1069,9 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
std::set<std::string> table_ids;
//remove to_delete files
try {
MetricCollector metric;
@ -1098,6 +1101,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_;
ConnectorPtr->remove<TableFileSchema>(table_file.id_);
table_ids.insert(table_file.table_id_);
}
return true;
});
@ -1111,6 +1115,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return HandleException("Encounter exception when clean table files", e);
}
//remove to_delete tables
try {
MetricCollector metric;
@ -1123,7 +1128,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto commited = ConnectorPtr->transaction([&]() mutable {
for (auto &table : tables) {
utils::DeleteTablePath(options_, std::get<1>(table));
utils::DeleteTablePath(options_, std::get<1>(table), false);//only delete empty folder
ConnectorPtr->remove<TableSchema>(std::get<0>(table));
}
@ -1139,6 +1144,23 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
return HandleException("Encounter exception when clean table files", e);
}
//remove deleted table folder
//don't remove table folder until all its files has been deleted
try {
MetricCollector metric;
for(auto& table_id : table_ids) {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_),
where(c(&TableFileSchema::table_id_) == table_id));
if(selected.size() == 0) {
utils::DeleteTablePath(options_, table_id);
}
}
} catch (std::exception &e) {
return HandleException("Encounter exception when delete table folder", e);
}
return Status::OK();
}

7
cpp/unittest/db/db_tests.cpp
View File

@ -93,6 +93,8 @@ TEST_F(DBTest, CONFIG_TEST) {
TEST_F(DBTest, DB_TEST) {
db_->Open(GetOptions(), &db_);
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
@ -161,6 +163,11 @@ TEST_F(DBTest, DB_TEST) {
}
search.join();
uint64_t count;
stat = db_->GetTableRowCount(TABLE_NAME, count);
ASSERT_STATS(stat);
ASSERT_TRUE(count > 0);
};
TEST_F(DBTest, SEARCH_TEST) {

39
cpp/unittest/db/mem_test.cpp
View File

@ -445,4 +445,41 @@ TEST_F(DBTest, VECTOR_IDS_TEST)
for (auto i = 0; i < nb; i++) {
ASSERT_EQ(vector_ids[i], i + nb);
}
}
}
TEST_F(NewMemManagerTest, MEMMANAGER_TEST) {
int setenv_res = setenv("MILVUS_USE_OLD_MEM_MANAGER", "ON", 1);
ASSERT_TRUE(setenv_res == 0);
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
auto start_time = METRICS_NOW_TIME;
int insert_loop = 20;
for (int i = 0; i < insert_loop; ++i) {
int64_t nb = 40960;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_TRUE(status.ok());
}
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
delete db_;
boost::filesystem::remove_all(options.meta.path);
}

7
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -194,6 +194,13 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
i++;
}
bool has;
status = impl.HasNonIndexFiles(table_id, has);
ASSERT_TRUE(status.ok());
status = impl.UpdateTableFilesToIndex(table_id);
ASSERT_TRUE(status.ok());
status = impl.DropAll();
ASSERT_TRUE(status.ok());
}

1
cpp/unittest/db/scheduler_test.cpp
View File

@ -56,6 +56,7 @@ TEST(DBSchedulerTest, TASK_QUEUE_TEST) {
ptr = queue.Back();
ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad);
load_task->Execute();
}
TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) {

3
cpp/unittest/server/config_test.cpp
View File

@ -87,6 +87,9 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = server::ServerConfig::GetInstance().ValidateConfig();
ASSERT_EQ(err, server::SERVER_SUCCESS);
server::ConfigNode node1 = config.GetConfig("server_config");
server::ConfigNode& node2 = config.GetConfig("cache_config");
node1.Combine(node2);

26
cpp/unittest/server/util_test.cpp
View File

@ -178,12 +178,15 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) {
res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
table_name = "_!@#!@";
res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
table_name = "中文";
res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
table_name = std::string('a', 32768);
table_name = std::string(10000, 'a');
res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
}
@ -204,3 +207,22 @@ TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE);
}
TEST(ValidationUtilTest, ValidateGpuTest) {
ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS);
size_t memory = 0;
ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS);
}
TEST(UtilTest, TIMERECORDER_TEST) {
for(int64_t log_level = 0; log_level <= 6; log_level++) {
if(log_level == 5) {
continue; //skip fatal
}
server::TimeRecorder rc("time", log_level);
rc.RecordSection("end");
}
}