admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-03 09:22:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into 'branch-0.5.0'

Browse Source 
MS-578 makesure milvus5.0 dont crack 0.3.1 data

See merge request megasearch/milvus!596

Former-commit-id: 61fd0fd041c9979ca7d9f761681bd1b80612318c
This commit is contained in:
jinhai 2019-09-20 13:01:39 +08:00
commit ae428c6ff6
22 changed files with 179 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-562 - Add JobMgr and TaskCreator in Scheduler
- MS-566 - Refactor cmake
- MS-555 - Remove old scheduler
- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data
## New Feature

14
cpp/README.md
View File

@ -68,16 +68,10 @@ $ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
###### Step 2 Build
```shell
TBD
cd [Milvus sourcecode path]/cpp/thirdparty
git clone git@192.168.1.105:megasearch/knowhere.git
cd knowhere
./build.sh -t Debug
or ./build.sh -t Release
cd [sourcecode path]/cpp
./build.sh -t Debug
or ./build.sh -t Release
$ cd [Milvus sourcecode path]/cpp
$ ./build.sh -t Debug
or
$ ./build.sh -t Release
```
When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`.

20
cpp/src/db/Options.cpp
View File

@ -17,7 +17,7 @@
#include "Options.h"
#include "utils/Exception.h"
#include "utils/easylogging++.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <assert.h>
@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
std::vector<std::string> kv;
boost::algorithm::split(kv, token, boost::is_any_of(":"));
if (kv.size() != 2) {
LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
continue;
}
if (kv[0] != "disk" && kv[0] != "days") {
LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
continue;
}
try {
@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
criterias_[kv[0]] = value;
}
catch (std::out_of_range&){
LOG(ERROR) << "Out of range: '" << kv[1] << "'";
throw OutOfRangeException();
std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
catch (...){
LOG(ERROR) << "Invalid argument: '" << kv[1] << "'";
throw InvalidArgumentException();
std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
}
}
void ArchiveConf::ParseType(const std::string& type) {
if (type != "delete" && type != "swap") {
LOG(ERROR) << "Invalid argument: type='" << type << "'";
throw InvalidArgumentException();
std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg);
}
type_ = type;
}

12
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
nlist_(nlist) {
index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
if (!index_) throw Exception("Create Empty VecIndex");
if (!index_) {
throw Exception(DB_ERROR, "Could not create VecIndex");
}
Config build_cfg;
build_cfg["dim"] = dimension;
build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
AutoGenParams(index_->GetType(), 0, build_cfg);
auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
if (ec != KNOWHERE_SUCCESS) {
throw Exception(DB_ERROR, "Build index error");
}
}
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
auto to_index = CreatetVecIndex(engine_type);
if (!to_index) {
throw Exception("Create Empty VecIndex");
throw Exception(DB_ERROR, "Could not create VecIndex");
}
Config build_cfg;
@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
from_index->GetRawVectors(),
from_index->GetRawIds(),
build_cfg);
if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
}

3
cpp/src/db/meta/Meta.h
View File

@ -31,6 +31,9 @@ namespace milvus {
namespace engine {
namespace meta {
static const char* META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles";
class Meta {
public:
virtual ~Meta() = default;

147
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -153,8 +153,9 @@ Status MySQLMetaImpl::Initialize() {
}
Query InitializeQuery = connectionPtr->query();
InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" <<
"id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
InitializeQuery << "CREATE TABLE IF NOT EXISTS " <<
META_TABLES << " " <<
"(id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
"table_id VARCHAR(255) UNIQUE NOT NULL, " <<
"state INT NOT NULL, " <<
"dimension SMALLINT NOT NULL, " <<
@ -171,8 +172,9 @@ Status MySQLMetaImpl::Initialize() {
return HandleException("Initialization Error", InitializeQuery.error());
}
InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" <<
"id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
InitializeQuery << "CREATE TABLE IF NOT EXISTS " <<
META_TABLEFILES << " " <<
"(id BIGINT PRIMARY KEY AUTO_INCREMENT, " <<
"table_id VARCHAR(255) NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
"file_id VARCHAR(255) NOT NULL, " <<
@ -233,7 +235,8 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id,
Query dropPartitionsByDatesQuery = connectionPtr->query();
dropPartitionsByDatesQuery << "UPDATE TableFiles " <<
dropPartitionsByDatesQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," <<
"updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
@ -266,7 +269,8 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
if (table_schema.table_id_.empty()) {
NextTableId(table_schema.table_id_);
} else {
createTableQuery << "SELECT state FROM Tables " <<
createTableQuery << "SELECT state FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << table_schema.table_id_ << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
@ -297,8 +301,9 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) {
std::string nlist = std::to_string(table_schema.nlist_);
std::string metric_type = std::to_string(table_schema.metric_type_);
createTableQuery << "INSERT INTO Tables VALUES" <<
"(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " <<
createTableQuery << "INSERT INTO " <<
META_TABLES << " " <<
"VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " <<
created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " <<
nlist << ", " << metric_type << ");";
@ -348,7 +353,8 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id,
Query hasNonIndexFilesQuery = connectionPtr->query();
//since table_id is a unique column we just need to check whether it exists or not
hasNonIndexFilesQuery << "SELECT file_id, file_type FROM TableFiles " <<
hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type in (" << types << ");";
@ -418,8 +424,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI
}
Query updateTableIndexParamQuery = connectionPtr->query();
updateTableIndexParamQuery << "SELECT id, state, dimension, created_on " <<
"FROM Tables " <<
updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
@ -435,7 +441,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI
uint16_t dimension = resRow["dimension"];
int64_t created_on = resRow["created_on"];
updateTableIndexParamQuery << "UPDATE Tables " <<
updateTableIndexParamQuery << "UPDATE " <<
META_TABLES << " " <<
"SET id = " << id << ", " <<
"state = " << state << ", " <<
"dimension = " << dimension << ", " <<
@ -476,7 +483,8 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag)
}
Query updateTableFlagQuery = connectionPtr->query();
updateTableFlagQuery << "UPDATE Tables " <<
updateTableFlagQuery << "UPDATE " <<
META_TABLES << " " <<
"SET flag = " << flag << " " <<
"WHERE table_id = " << quote << table_id << ";";
@ -507,8 +515,8 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex
}
Query describeTableIndexQuery = connectionPtr->query();
describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type " <<
"FROM Tables " <<
describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
@ -549,7 +557,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
Query dropTableIndexQuery = connectionPtr->query();
//soft delete index files
dropTableIndexQuery << "UPDATE TableFiles " <<
dropTableIndexQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," <<
"updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
@ -562,7 +571,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
}
//set all backup file to raw
dropTableIndexQuery << "UPDATE TableFiles " <<
dropTableIndexQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::RAW) << "," <<
"updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
@ -575,7 +585,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
}
//set table index type to raw
dropTableIndexQuery << "UPDATE Tables " <<
dropTableIndexQuery << "UPDATE " <<
META_TABLES << " " <<
"SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," <<
"nlist = " << std::to_string(DEFAULT_NLIST) << ", " <<
"metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " <<
@ -609,7 +620,8 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) {
//soft delete table
Query deleteTableQuery = connectionPtr->query();
//
deleteTableQuery << "UPDATE Tables " <<
deleteTableQuery << "UPDATE " <<
META_TABLES << " " <<
"SET state = " << std::to_string(TableSchema::TO_DELETE) << " " <<
"WHERE table_id = " << quote << table_id << ";";
@ -645,7 +657,8 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) {
//soft delete table files
Query deleteTableFilesQuery = connectionPtr->query();
//
deleteTableFilesQuery << "UPDATE TableFiles " <<
deleteTableFilesQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " <<
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
@ -676,9 +689,8 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
}
Query describeTableQuery = connectionPtr->query();
describeTableQuery << "SELECT id, state, dimension, created_on, " <<
"flag, index_file_size, engine_type, nlist, metric_type " <<
"FROM Tables " <<
describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << table_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
@ -732,7 +744,8 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
Query hasTableQuery = connectionPtr->query();
//since table_id is a unique column we just need to check whether it exists or not
hasTableQuery << "SELECT EXISTS " <<
"(SELECT 1 FROM Tables " <<
"(SELECT 1 FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << table_id << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
@ -764,8 +777,8 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
}
Query allTablesQuery = connectionPtr->query();
allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type " <<
"FROM Tables " <<
allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " <<
META_TABLES << " " <<
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str();
@ -846,8 +859,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
Query createTableFileQuery = connectionPtr->query();
createTableFileQuery << "INSERT INTO TableFiles VALUES" <<
"(" << id << ", " << quote << table_id << ", " << engine_type << ", " <<
createTableFileQuery << "INSERT INTO " <<
META_TABLEFILES << " " <<
"VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " <<
quote << file_id << ", " << file_type << ", " << file_size << ", " <<
row_count << ", " << updated_time << ", " << created_on << ", " << date << ");";
@ -883,8 +897,8 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
}
Query filesToIndexQuery = connectionPtr->query();
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on " <<
"FROM TableFiles " <<
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " <<
META_TABLEFILES << " " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str();
@ -967,8 +981,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
}
Query filesToSearchQuery = connectionPtr->query();
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " <<
"FROM TableFiles " <<
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id;
if (!partition.empty()) {
@ -1086,8 +1100,8 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
}
Query filesToMergeQuery = connectionPtr->query();
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " <<
"FROM TableFiles " <<
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << " " <<
"ORDER BY row_count DESC" << ";";
@ -1177,8 +1191,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
}
Query getTableFileQuery = connectionPtr->query();
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " <<
"FROM TableFiles " <<
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ") AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
@ -1259,7 +1273,8 @@ Status MySQLMetaImpl::Archive() {
}
Query archiveQuery = connectionPtr->query();
archiveQuery << "UPDATE TableFiles " <<
archiveQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"WHERE created_on < " << std::to_string(now - usecs) << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
@ -1299,8 +1314,8 @@ Status MySQLMetaImpl::Size(uint64_t &result) {
}
Query getSizeQuery = connectionPtr->query();
getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " <<
"FROM TableFiles " <<
getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " <<
META_TABLEFILES << " " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str();
@ -1339,8 +1354,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) {
}
Query discardFilesQuery = connectionPtr->query();
discardFilesQuery << "SELECT id, file_size " <<
"FROM TableFiles " <<
discardFilesQuery << "SELECT id, file_size FROM " <<
META_TABLEFILES << " " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " <<
"ORDER BY id ASC " <<
"LIMIT 10;";
@ -1369,7 +1384,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) {
std::string idsToDiscardStr = idsToDiscardSS.str();
idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR "
discardFilesQuery << "UPDATE TableFiles " <<
discardFilesQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " <<
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE " << idsToDiscardStr << ";";
@ -1406,7 +1422,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
//if the table has been deleted, just mark the table file as TO_DELETE
//clean thread will delete the file later
updateTableFileQuery << "SELECT state FROM Tables " <<
updateTableFileQuery << "SELECT state FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << file_schema.table_id_ << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
@ -1433,7 +1450,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
updateTableFileQuery << "UPDATE TableFiles " <<
updateTableFileQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET table_id = " << quote << table_id << ", " <<
"engine_type = " << engine_type << ", " <<
"file_id = " << quote << file_id << ", " <<
@ -1470,7 +1488,8 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) {
Query updateTableFilesToIndexQuery = connectionPtr->query();
updateTableFilesToIndexQuery << "UPDATE TableFiles " <<
updateTableFilesToIndexQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << ";";
@ -1508,7 +1527,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
}
updateTableFilesQuery << "SELECT EXISTS " <<
"(SELECT 1 FROM Tables " <<
"(SELECT 1 FROM " <<
META_TABLES << " " <<
"WHERE table_id = " << quote << file_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
@ -1539,7 +1559,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
std::string created_on = std::to_string(file_schema.created_on_);
std::string date = std::to_string(file_schema.date_);
updateTableFilesQuery << "UPDATE TableFiles " <<
updateTableFilesQuery << "UPDATE " <<
META_TABLEFILES << " " <<
"SET table_id = " << quote << table_id << ", " <<
"engine_type = " << engine_type << ", " <<
"file_id = " << quote << file_id << ", " <<
@ -1582,8 +1603,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
Query cleanUpFilesWithTTLQuery = connectionPtr->query();
cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " <<
"FROM TableFiles " <<
cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " <<
META_TABLEFILES << " " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " <<
"updated_time < " << std::to_string(now - seconds * US_PS) << ";";
@ -1626,8 +1647,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " <<
idsToDeleteStr << ";";
cleanUpFilesWithTTLQuery << "DELETE FROM " <<
META_TABLEFILES << " " <<
"WHERE " << idsToDeleteStr << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
@ -1653,8 +1675,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
Query cleanUpFilesWithTTLQuery = connectionPtr->query();
cleanUpFilesWithTTLQuery << "SELECT id, table_id " <<
"FROM Tables " <<
cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " <<
META_TABLES << " " <<
"WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
@ -1675,8 +1697,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR "
cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " <<
idsToDeleteStr << ";";
cleanUpFilesWithTTLQuery << "DELETE FROM " <<
META_TABLES << " " <<
"WHERE " << idsToDeleteStr << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
@ -1704,8 +1727,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
for(auto& table_id : table_ids) {
Query cleanUpFilesWithTTLQuery = connectionPtr->query();
cleanUpFilesWithTTLQuery << "SELECT file_id " <<
"FROM TableFiles " <<
cleanUpFilesWithTTLQuery << "SELECT file_id FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
@ -1736,7 +1759,7 @@ Status MySQLMetaImpl::CleanUp() {
cleanUpQuery << "SELECT table_name " <<
"FROM information_schema.tables " <<
"WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " <<
"AND table_name = " << quote << "TableFiles" << ";";
"AND table_name = " << quote << META_TABLEFILES << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
@ -1744,7 +1767,7 @@ Status MySQLMetaImpl::CleanUp() {
if (!res.empty()) {
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type IN ("
cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN ("
<< std::to_string(TableFileSchema::NEW) << ","
<< std::to_string(TableFileSchema::NEW_MERGE) << ","
<< std::to_string(TableFileSchema::NEW_INDEX) << ");";
@ -1785,8 +1808,8 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) {
Query countQuery = connectionPtr->query();
countQuery << "SELECT row_count " <<
"FROM TableFiles " <<
countQuery << "SELECT row_count FROM " <<
META_TABLEFILES << " " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
@ -1820,7 +1843,7 @@ Status MySQLMetaImpl::DropAll() {
}
Query dropTableQuery = connectionPtr->query();
dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;";
dropTableQuery << "DROP TABLE IF EXISTS " << META_TABLES << ", " << META_TABLEFILES << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str();

20
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -19,6 +19,7 @@
#include "db/IDGenerator.h"
#include "db/Utils.h"
#include "utils/Log.h"
#include "utils/Exception.h"
#include "MetaConsts.h"
#include "metrics/Metrics.h"
@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) {
inline auto StoragePrototype(const std::string &path) {
return make_storage(path,
make_table("Tables",
make_table(META_TABLES,
make_column("id", &TableSchema::id_, primary_key()),
make_column("table_id", &TableSchema::table_id_, unique()),
make_column("state", &TableSchema::state_),
@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) {
make_column("engine_type", &TableSchema::engine_type_),
make_column("nlist", &TableSchema::nlist_),
make_column("metric_type", &TableSchema::metric_type_)),
make_table("TableFiles",
make_table(META_TABLEFILES,
make_column("id", &TableFileSchema::id_, primary_key()),
make_column("table_id", &TableFileSchema::table_id_),
make_column("engine_type", &TableFileSchema::engine_type_),
@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() {
ConnectorPtr = std::make_unique<ConnectorT>(StoragePrototype(options_.path + "/meta.sqlite"));
//old meta could be recreated since schema changed, throw exception if meta schema is not compatible
auto ret = ConnectorPtr->sync_schema_simulate();
if(ret.find(META_TABLES) != ret.end()
&& sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
}
if(ret.find(META_TABLEFILES) != ret.end()
&& sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) {
throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
}
ConnectorPtr->sync_schema();
ConnectorPtr->open_forever(); // thread safe option
ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() {
ENGINE_LOG_DEBUG << "Drop all sqlite meta";
try {
ConnectorPtr->drop_table("Tables");
ConnectorPtr->drop_table("TableFiles");
ConnectorPtr->drop_table(META_TABLES);
ConnectorPtr->drop_table(META_TABLEFILES);
} catch (std::exception &e) {
return HandleException("Encounter exception when drop all meta", e.what());
}

8
cpp/src/main.cpp
View File

@ -102,10 +102,6 @@ main(int argc, char *argv[]) {
}
}
server::Server &server = server::Server::Instance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start();
/* Handle Signal */
signal(SIGHUP, server::SignalUtil::HandleSignal);
signal(SIGINT, server::SignalUtil::HandleSignal);
@ -114,6 +110,10 @@ main(int argc, char *argv[]) {
signal(SIGUSR2, server::SignalUtil::HandleSignal);
signal(SIGTERM, server::SignalUtil::HandleSignal);
server::Server &server = server::Server::Instance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start();
/* wait signal */
pause();

7
cpp/src/server/DBWrapper.cpp
View File

@ -109,15 +109,10 @@ Status DBWrapper::StartService() {
}
//create db instance
std::string msg = opt.meta.path;
try {
db_ = engine::DBFactory::Build(opt);
} catch(std::exception& ex) {
msg = ex.what();
}
if(db_ == nullptr) {
std::cerr << "ERROR! Failed to open database: " << msg << std::endl;
std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl;
kill(0, SIGUSR1);
}

2
cpp/src/server/Server.cpp
View File

@ -196,8 +196,8 @@ Server::Start() {
server::Metrics::GetInstance().Init();
server::SystemInfo::GetInstance().Init();
std::cout << "Milvus server start successfully." << std::endl;
StartService();
std::cout << "Milvus server start successfully." << std::endl;
} catch (std::exception &ex) {
std::cerr << "Milvus server encounter exception: " << ex.what();

1
cpp/src/utils/Error.h
View File

@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
//knowhere error code
constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);

30
cpp/src/utils/Exception.h
View File

@ -17,6 +17,8 @@
#pragma once
#include "utils/Error.h"
#include <exception>
#include <string>
@ -25,13 +27,14 @@ namespace milvus {
class Exception : public std::exception {
public:
Exception(const std::string& message)
: message_(message) {
Exception(ErrorCode code, const std::string& message)
: code_(code_),
message_(message) {
}
Exception()
: message_() {
}
ErrorCode code() const throw() {
return code_;
}
virtual const char* what() const throw() {
if (message_.empty()) {
@ -44,20 +47,21 @@ public:
virtual ~Exception() throw() {};
protected:
ErrorCode code_;
std::string message_;
};
class InvalidArgumentException : public Exception {
public:
InvalidArgumentException() : Exception("Invalid Argument"){};
InvalidArgumentException(const std::string& message) : Exception(message) {};
};
InvalidArgumentException()
: Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
};
InvalidArgumentException(const std::string& message)
: Exception(SERVER_INVALID_ARGUMENT, message) {
};
class OutOfRangeException : public Exception {
public:
OutOfRangeException() : Exception("Out Of Range"){};
OutOfRangeException(const std::string& message) : Exception(message) {};
};
} // namespace milvus

3
cpp/src/utils/Log.h
View File

@ -24,7 +24,6 @@ namespace milvus {
/////////////////////////////////////////////////////////////////////////////////////////////////
#define SERVER_DOMAIN_NAME "[SERVER] "
#define SERVER_ERROR_TEXT "SERVER Error:"
#define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME
#define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME
@ -35,7 +34,6 @@ namespace milvus {
/////////////////////////////////////////////////////////////////////////////////////////////////
#define ENGINE_DOMAIN_NAME "[ENGINE] "
#define ENGINE_ERROR_TEXT "ENGINE Error:"
#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
@ -46,7 +44,6 @@ namespace milvus {
/////////////////////////////////////////////////////////////////////////////////////////////////
#define WRAPPER_DOMAIN_NAME "[WRAPPER] "
#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME

2
cpp/src/utils/LogUtil.cpp
View File

@ -17,10 +17,8 @@
#include "LogUtil.h"
#include "server/ServerConfig.h"
#include "easylogging++.h"
#include <ctype.h>
#include <string>
#include <libgen.h>

3
cpp/unittest/db/db_tests.cpp
View File

@ -24,10 +24,7 @@
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>
#include <thread>
#include <random>

11
cpp/unittest/db/meta_tests.cpp
View File

@ -15,17 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <thread>
#include "utils/easylogging++.h"
#include <stdlib.h>
#include <time.h>
#include "utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include <gtest/gtest.h>
#include <thread>
#include <stdlib.h>
#include <time.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;

11
cpp/unittest/db/misc_test.cpp
View File

@ -21,7 +21,6 @@
#include "db/Utils.h"
#include "utils/Status.h"
#include "utils/Exception.h"
#include "utils/easylogging++.h"
#include <gtest/gtest.h>
#include <thread>
@ -31,13 +30,13 @@
using namespace zilliz::milvus;
TEST(DBMiscTest, EXCEPTION_TEST) {
Exception ex1("");
Exception ex1(100, "error");
std::string what = ex1.what();
ASSERT_FALSE(what.empty());
ASSERT_EQ(what, "error");
ASSERT_EQ(ex1.code(), 100);
OutOfRangeException ex2;
what = ex2.what();
ASSERT_FALSE(what.empty());
InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT);
}
TEST(DBMiscTest, OPTIONS_TEST) {

1
cpp/unittest/db/mysql_db_test.cpp
View File

@ -21,7 +21,6 @@
#include "db/meta/MetaConsts.h"
#include <gtest/gtest.h>
#include "utils/easylogging++.h"
#include <boost/filesystem.hpp>
#include <thread>

13
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -15,20 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <thread>
#include "utils/easylogging++.h"
#include <stdlib.h>
#include <time.h>
#include "utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "mysql++/mysql++.h"
#include <iostream>
#include <thread>
#include <stdlib.h>
#include <time.h>
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;

6
cpp/unittest/main.cpp
View File

@ -15,13 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "utils/easylogging++.h"
#include "server/ServerConfig.h"
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include <gmock/gmock.h>
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;

15
cpp/unittest/server/util_test.cpp
View File

@ -15,14 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <thread>
#include "utils/easylogging++.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/filesystem.hpp>
#include <src/utils/SignalUtil.h>
#include "utils/SignalUtil.h"
#include "utils/CommonUtil.h"
#include "utils/Error.h"
#include "utils/StringHelpFunctions.h"
@ -32,6 +25,12 @@
#include "utils/ValidationUtil.h"
#include "db/engine/ExecutionEngine.h"
#include <thread>
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/filesystem.hpp>
#include <gtest/gtest.h>
using namespace zilliz::milvus;
namespace {

6
cpp/unittest/wrapper/wrapper_test.cpp
View File

@ -15,15 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "utils/easylogging++.h"
#include "src/wrapper/vec_index.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "utils.h"
#include <gtest/gtest.h>
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus::engine;