admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve codecov (#1095)

Browse Source 
* Optimize config test. Dir src/config 99% lines covered

* add unittest coverage

* optimize cache&config unittest

* code format

* format

* format code

* fix merge conflict

* cover src/utils unittest

*  '#831 fix exe_path judge error'

* #831 fix exe_path judge error

* add some unittest coverage

* add some unittest coverage

* improve coverage of src/wrapper

* improve src/wrapper coverage

* *test optimize db/meta unittest

* fix bug

* *test optimize mysqlMetaImpl unittest

* *style: format code

* import server& scheduler unittest coverage

* handover next work

* *test: add some test_meta test case

* *format code

* *fix: fix typo

* feat(codecov): improve code coverage for src/db(#872)

* feat(codecov): improve code coverage for src/db/engine(#872)

* feat(codecov): improve code coverage(#872)

* fix config unittest bug

* feat(codecov): improve code coverage core/db/engine(#872)

* feat(codecov): improve code coverage core/knowhere

* feat(codecov): improve code coverage core/knowhere

* feat(codecov): improve code coverage

* feat(codecov): fix cpu test some error

* feat(codecov): improve code coverage

* feat(codecov): rename some fiu

* fix(db/meta): fix switch/case default action

* feat(codecov): improve code coverage(#872)
* fix error caused by merge code
* format code

* feat(codecov): improve code coverage & format code(#872)

* feat(codecov): fix test error(#872)

* feat(codecov): fix unittest test_mem(#872)

* feat(codecov): fix unittest(#872)

* feat(codecov): fix unittest for resource manager(#872)

* feat(codecov): code format (#872)

* feat(codecov): trigger ci(#872)

* fix(RequestScheduler): remove a wrong sleep statement

* test(test_rpc): fix rpc test

* Fix format issue

* Remove unused comments

* Fix unit test error

Co-authored-by: ABNER-1 <ABNER-1@users.noreply.github.com>
Co-authored-by: Jin Hai <hai.jin@zilliz.com>
This commit is contained in:
shengjh 2020-02-11 23:03:29 +08:00 committed by GitHub
parent cd800042a5
commit 4dee7dfac6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
98 changed files with 4081 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/cache/CpuCacheMgr.cpp vendored
View File

@ -19,6 +19,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <utility>
namespace milvus {

1
core/src/cache/GpuCacheMgr.cpp vendored
View File

@ -19,6 +19,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <sstream>
#include <utility>

1
core/src/config/ConfigNode.cpp
View File

@ -19,6 +19,7 @@
#include "utils/Error.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <algorithm>
#include <sstream>
#include <string>

22
core/src/db/DBImpl.cpp
View File

@ -220,17 +220,20 @@ DBImpl::PreloadTable(const std::string& table_id) {
for (auto& file : files_array) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
fiu_do_on("DBImpl.PreloadTable.null_engine", engine = nullptr);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
size += engine->PhysicalSize();
fiu_do_on("DBImpl.PreloadTable.exceed_cache", size = available_size + 1);
if (size > available_size) {
ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full";
return Status(SERVER_CACHE_FULL, "Cache is full");
} else {
try {
fiu_do_on("DBImpl.PreloadTable.engine_throw_exception", throw std::exception());
std::string msg = "Pre-loaded file: " + file.file_id_ + " size: " + std::to_string(file.file_size_);
TimeRecorderAuto rc_1(msg);
engine->Load(true);
@ -492,6 +495,7 @@ DBImpl::QueryByFileID(const std::shared_ptr<server::Context>& context, const std
return status;
}
fiu_do_on("DBImpl.QueryByFileID.empty_files_array", files_array.clear());
if (files_array.empty()) {
return Status(DB_ERROR, "Invalid file id");
}
@ -601,11 +605,11 @@ DBImpl::StartMetricTask() {
return;
}
// ENGINE_LOG_TRACE << "Start metric task";
server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
fiu_do_on("DBImpl.StartMetricTask.InvalidTotalCache", cache_total = 0);
if (cache_total > 0) {
double cache_usage_double = cache_usage;
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage_double * 100 / cache_total);
@ -627,8 +631,6 @@ DBImpl::StartMetricTask() {
server::Metrics::GetInstance().GPUTemperature();
server::Metrics::GetInstance().CPUTemperature();
server::Metrics::GetInstance().PushToGateway();
// ENGINE_LOG_TRACE << "Metric task finished";
}
Status
@ -733,6 +735,8 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m
// step 3: serialize to disk
try {
status = index->Serialize();
fiu_do_on("DBImpl.MergeFiles.Serialize_ThrowException", throw std::exception());
fiu_do_on("DBImpl.MergeFiles.Serialize_ErrorStatus", status = Status(DB_ERROR, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
}
@ -987,6 +991,7 @@ DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& da
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableRecursively(schema.table_id_, dates);
fiu_do_on("DBImpl.DropTableRecursively.failed", status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -1000,6 +1005,8 @@ DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableInde
DropIndex(table_id);
auto status = meta_ptr_->UpdateTableIndex(table_id, index);
fiu_do_on("DBImpl.UpdateTableIndexRecursively.fail_update_table_index",
status = Status(DB_META_TRANSACTION_FAILED, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id;
return status;
@ -1060,6 +1067,8 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = BuildTableIndexRecursively(schema.table_id_, index);
fiu_do_on("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -1068,6 +1077,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
// failed to build index for some files, return error
std::string err_msg;
index_failed_checker_.GetErrMsgForTable(table_id, err_msg);
fiu_do_on("DBImpl.BuildTableIndexRecursively.not_empty_err_msg", err_msg.append("fiu"));
if (!err_msg.empty()) {
return Status(DB_ERROR, err_msg);
}
@ -1089,6 +1099,8 @@ DBImpl::DropTableIndexRecursively(const std::string& table_id) {
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableIndexRecursively(schema.table_id_);
fiu_do_on("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -1111,6 +1123,8 @@ DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_c
for (auto& schema : partition_array) {
uint64_t partition_row_count = 0;
status = GetTableRowCountRecursively(schema.table_id_, partition_row_count);
fiu_do_on("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition",
status = Status(DB_ERROR, ""));
if (!status.ok()) {
return status;
}

1
core/src/db/IDGenerator.cpp
View File

@ -18,6 +18,7 @@
#include "db/IDGenerator.h"
#include <assert.h>
#include <fiu-local.h>
#include <chrono>
#include <iostream>

5
core/src/db/Options.cpp
View File

@ -16,6 +16,8 @@
// under the License.
#include "db/Options.h"
#include <fiu-local.h>
#include <limits>
#include "utils/Exception.h"
#include "utils/Log.h"
@ -45,6 +47,7 @@ ArchiveConf::ParseCritirias(const std::string& criterias) {
boost::algorithm::split(tokens, criterias, boost::is_any_of(";"));
fiu_do_on("ArchiveConf.ParseCritirias.empty_tokens", tokens.clear());
if (tokens.size() == 0) {
return;
}
@ -65,6 +68,8 @@ ArchiveConf::ParseCritirias(const std::string& criterias) {
continue;
}
try {
fiu_do_on("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange",
kv[1] = std::to_string(std::numeric_limits<int>::max() + 1UL));
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
} catch (std::out_of_range&) {

4
core/src/db/Utils.cpp
View File

@ -21,6 +21,7 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <chrono>
#include <mutex>
@ -94,6 +95,7 @@ CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
for (auto& path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
fiu_do_on("CreateTablePath.creat_slave_path", status = Status(DB_INVALID_PATH, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
@ -141,6 +143,7 @@ CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_f
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
fiu_do_on("CreateTableFilePath.fail_create", status = Status(DB_INVALID_PATH, ""));
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
@ -159,6 +162,7 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);
fiu_do_on("GetTableFilePath.enable_s3", s3_enable = true);
if (s3_enable) {
/* need not check file existence */
table_file.location_ = file_path;

4
core/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -17,6 +17,7 @@
#include "db/engine/ExecutionEngineImpl.h"
#include <fiu-local.h>
#include <stdexcept>
#include <utility>
#include <vector>
@ -139,7 +140,10 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
server::Config& config = server::Config::GetInstance();
bool gpu_resource_enable = true;
config.GetGpuResourceConfigEnable(gpu_resource_enable);
fiu_do_on("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled", gpu_resource_enable = false);
#endif
fiu_do_on("ExecutionEngineImpl.CreatetVecIndex.invalid_type", type = EngineType::INVALID);
std::shared_ptr<VecIndex> index;
switch (type) {
case EngineType::FAISS_IDMAP: {

3
core/src/db/meta/MySQLConnectionPool.cpp
View File

@ -16,6 +16,7 @@
// under the License.
#include "db/meta/MySQLConnectionPool.h"
#include <fiu-local.h>
namespace milvus {
namespace engine {
@ -65,6 +66,8 @@ MySQLConnectionPool::getDB() {
mysqlpp::Connection*
MySQLConnectionPool::create() {
try {
fiu_do_on("MySQLConnectionPool.create.throw_exception", throw mysqlpp::ConnectionFailed());
// Create connection using the parameters we were passed upon
// creation.
auto conn = new mysqlpp::Connection();

186
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -25,6 +25,7 @@
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <fiu-local.h>
#include <mysql++/mysql++.h>
#include <string.h>
#include <unistd.h>
@ -209,6 +210,7 @@ MySQLMetaImpl::ValidateMetaSchema() {
}
auto validate_func = [&](const MetaSchema& schema) {
fiu_return_on("MySQLMetaImpl.ValidateMetaSchema.fail_validate", false);
mysqlpp::Query query_statement = connectionPtr->query();
query_statement << "DESC " << schema.name() << ";";
@ -251,6 +253,7 @@ MySQLMetaImpl::Initialize() {
// step 1: create db root path
if (!boost::filesystem::is_directory(options_.path_)) {
auto ret = boost::filesystem::create_directory(options_.path_);
fiu_do_on("MySQLMetaImpl.Initialize.fail_create_directory", ret = false);
if (!ret) {
std::string msg = "Failed to create db directory " + options_.path_;
ENGINE_LOG_ERROR << msg;
@ -304,7 +307,9 @@ MySQLMetaImpl::Initialize() {
throw Exception(DB_INVALID_META_URI, msg);
}
if (!connectionPtr->thread_aware()) {
bool is_thread_aware = connectionPtr->thread_aware();
fiu_do_on("MySQLMetaImpl.Initialize.is_thread_aware", is_thread_aware = false);
if (!is_thread_aware) {
std::string msg =
"Failed to initialize MySQL meta backend: MySQL client component wasn't built with thread awareness";
ENGINE_LOG_ERROR << msg;
@ -318,7 +323,9 @@ MySQLMetaImpl::Initialize() {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
bool initialize_query_exec = InitializeQuery.exec();
fiu_do_on("MySQLMetaImpl.Initialize.fail_create_table_scheme", initialize_query_exec = false);
if (!initialize_query_exec) {
std::string msg = "Failed to create meta table 'Tables' in MySQL";
ENGINE_LOG_ERROR << msg;
throw Exception(DB_META_TRANSACTION_FAILED, msg);
@ -330,7 +337,9 @@ MySQLMetaImpl::Initialize() {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
initialize_query_exec = InitializeQuery.exec();
fiu_do_on("MySQLMetaImpl.Initialize.fail_create_table_files", initialize_query_exec = false);
if (!initialize_query_exec) {
std::string msg = "Failed to create meta table 'TableFiles' in MySQL";
ENGINE_LOG_ERROR << msg;
throw Exception(DB_META_TRANSACTION_FAILED, msg);
@ -346,7 +355,10 @@ MySQLMetaImpl::CreateTable(TableSchema& table_schema) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CreateTable.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CreateTable.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -364,6 +376,7 @@ MySQLMetaImpl::CreateTable(TableSchema& table_schema) {
if (res.num_rows() == 1) {
int state = res[0]["state"];
fiu_do_on("MySQLMetaImpl.CreateTableTable.schema_TO_DELETE", state = TableSchema::TO_DELETE);
if (TableSchema::TO_DELETE == state) {
return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second");
} else {
@ -421,7 +434,10 @@ MySQLMetaImpl::DescribeTable(TableSchema& table_schema) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DescribeTable.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DescribeTable.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -469,7 +485,10 @@ MySQLMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.HasTable.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.HasTable.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -503,7 +522,10 @@ MySQLMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.AllTable.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.AllTable.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -547,7 +569,11 @@ MySQLMetaImpl::DropTable(const std::string& table_id) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DropTable.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DropTable.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -564,7 +590,9 @@ MySQLMetaImpl::DropTable(const std::string& table_id) {
}
} // Scoped Connection
if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
bool is_writable_mode{mode_ == DBOptions::MODE::CLUSTER_WRITABLE};
fiu_do_on("MySQLMetaImpl.DropTable.CLUSTER_WRITABLE_MODE", is_writable_mode = true);
if (is_writable_mode) {
DeleteTableFiles(table_id);
}
@ -583,7 +611,11 @@ MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DeleteTableFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DeleteTableFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -651,7 +683,10 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CreateTableFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CreateTableFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -705,7 +740,10 @@ MySQLMetaImpl::DropDataByDate(const std::string& table_id, const DatesT& dates)
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DropDataByDate.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DropDataByDate.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -751,7 +789,10 @@ MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector<size
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.GetTableFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.GetTableFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -806,7 +847,10 @@ MySQLMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& i
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.UpdateTableIndex.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.UpdateTableIndex.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -860,7 +904,10 @@ MySQLMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.UpdateTableFlag.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.UpdateTableFlag.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -893,7 +940,10 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema& file_schema) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.UpdateTableFile.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.UpdateTableFile.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -956,7 +1006,10 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
try {
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.UpdateTableFilesToIndex.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.UpdateTableFilesToIndex.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -990,7 +1043,10 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.UpdateTableFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.UpdateTableFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1065,7 +1121,10 @@ MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DescribeTableIndex.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DescribeTableIndex.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1103,7 +1162,10 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DropTableIndex.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DropTableIndex.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1198,6 +1260,7 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p
table_schema.partition_tag_ = valid_tag;
status = CreateTable(table_schema);
fiu_do_on("MySQLMetaImpl.CreatePartition.aleady_exist", status = Status(DB_ALREADY_EXIST, ""));
if (status.code() == DB_ALREADY_EXIST) {
return Status(DB_ALREADY_EXIST, "Partition already exists");
}
@ -1218,7 +1281,10 @@ MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::Tab
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.ShowPartitions.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.ShowPartitions.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1258,7 +1324,10 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string&
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.GetPartitionName.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.GetPartitionName.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1296,7 +1365,10 @@ MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector<size
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.FilesToSearch.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.FilesToSearch.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1400,7 +1472,10 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.FilesToMerge.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.FilesToMerge.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1470,7 +1545,10 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.FilesToIndex.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.FilesToIndex.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1545,7 +1623,10 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.FilesByType.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.FilesByType.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1610,7 +1691,7 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
++backup_count;
break;
default:
return Status(DB_ERROR, "Unknown file type.");
break;
}
}
@ -1639,7 +1720,7 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
msg = msg + " backup files:" + std::to_string(backup_count);
break;
default:
return Status(DB_ERROR, "Unknown file type!");
break;
}
}
ENGINE_LOG_DEBUG << msg;
@ -1669,7 +1750,10 @@ MySQLMetaImpl::Archive() {
try {
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.Archive.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.Archive.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1713,7 +1797,10 @@ MySQLMetaImpl::Size(uint64_t& result) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.Size.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.Size.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1744,7 +1831,10 @@ MySQLMetaImpl::CleanUpShadowFiles() {
try {
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CleanUpShadowFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CleanUpShadowFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1793,7 +1883,11 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_NullConnection",
is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_ThrowException", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1875,7 +1969,11 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_NUllConnection",
is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_ThrowException", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1926,7 +2024,12 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_NUllConnection",
is_null_connection = true);
fiu_do_on("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_ThrowException",
throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -1972,7 +2075,10 @@ MySQLMetaImpl::Count(const std::string& table_id, uint64_t& result) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.Count.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.Count.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -2006,7 +2112,10 @@ MySQLMetaImpl::DropAll() {
ENGINE_LOG_DEBUG << "Drop all mysql meta";
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DropAll.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DropAll.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
@ -2037,7 +2146,10 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) {
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
if (connectionPtr == nullptr) {
bool is_null_connection = (connectionPtr == nullptr);
fiu_do_on("MySQLMetaImpl.DiscardFiles.null_connection", is_null_connection = true);
fiu_do_on("MySQLMetaImpl.DiscardFiles.throw_exception", throw std::exception(););
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}

89
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -35,6 +35,7 @@
#include <memory>
#include <set>
#include <sstream>
#include <fiu-local.h>
namespace milvus {
namespace engine {
@ -120,7 +121,9 @@ SqliteMetaImpl::NextFileId(std::string& file_id) {
void
SqliteMetaImpl::ValidateMetaSchema() {
if (ConnectorPtr == nullptr) {
bool is_null_connector{ConnectorPtr == nullptr};
fiu_do_on("SqliteMetaImpl.ValidateMetaSchema.NullConnection", is_null_connector = true);
if (is_null_connector) {
return;
}
@ -140,6 +143,7 @@ Status
SqliteMetaImpl::Initialize() {
if (!boost::filesystem::is_directory(options_.path_)) {
auto ret = boost::filesystem::create_directory(options_.path_);
fiu_do_on("SqliteMetaImpl.Initialize.fail_create_directory", ret = false);
if (!ret) {
std::string msg = "Failed to create db directory " + options_.path_;
ENGINE_LOG_ERROR << msg;
@ -171,6 +175,7 @@ SqliteMetaImpl::CreateTable(TableSchema& table_schema) {
if (table_schema.table_id_ == "") {
NextTableId(table_schema.table_id_);
} else {
fiu_do_on("SqliteMetaImpl.CreateTable.throw_exception", throw std::exception());
auto table = ConnectorPtr->select(columns(&TableSchema::state_),
where(c(&TableSchema::table_id_) == table_schema.table_id_));
if (table.size() == 1) {
@ -187,6 +192,7 @@ SqliteMetaImpl::CreateTable(TableSchema& table_schema) {
table_schema.created_on_ = utils::GetMicroSecTimeStamp();
try {
fiu_do_on("SqliteMetaImpl.CreateTable.insert_throw_exception", throw std::exception());
auto id = ConnectorPtr->insert(table_schema);
table_schema.id_ = id;
} catch (std::exception& e) {
@ -206,6 +212,7 @@ SqliteMetaImpl::DescribeTable(TableSchema& table_schema) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DescribeTable.throw_exception", throw std::exception());
auto groups = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::state_,
&TableSchema::dimension_,
@ -249,6 +256,7 @@ SqliteMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
has_or_not = false;
try {
fiu_do_on("SqliteMetaImpl.HasTable.throw_exception", throw std::exception());
server::MetricCollector metric;
auto tables = ConnectorPtr->select(columns(&TableSchema::id_),
where(c(&TableSchema::table_id_) == table_id
@ -268,8 +276,8 @@ SqliteMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
Status
SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
try {
fiu_do_on("SqliteMetaImpl.AllTables.throw_exception", throw std::exception());
server::MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::table_id_,
&TableSchema::dimension_,
@ -310,6 +318,8 @@ SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
Status
SqliteMetaImpl::DropTable(const std::string& table_id) {
try {
fiu_do_on("SqliteMetaImpl.DropTable.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@ -334,6 +344,8 @@ SqliteMetaImpl::DropTable(const std::string& table_id) {
Status
SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) {
try {
fiu_do_on("SqliteMetaImpl.DeleteTableFiles.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@ -369,6 +381,7 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) {
}
try {
fiu_do_on("SqliteMetaImpl.CreateTableFile.throw_exception", throw std::exception());
server::MetricCollector metric;
NextFileId(file_schema.file_id_);
@ -413,6 +426,8 @@ SqliteMetaImpl::DropDataByDate(const std::string& table_id,
}
try {
fiu_do_on("SqliteMetaImpl.DropDataByDate.throw_exception", throw std::exception());
// sqlite_orm has a bug, 'in' statement cannot handle too many elements
// so we split one query into multi-queries, this is a work-around!!
std::vector<DatesT> split_dates;
@ -453,6 +468,8 @@ SqliteMetaImpl::GetTableFiles(const std::string& table_id,
const std::vector<size_t>& ids,
TableFilesSchema& table_files) {
try {
fiu_do_on("SqliteMetaImpl.GetTableFiles.throw_exception", throw std::exception());
table_files.clear();
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::file_id_,
@ -505,6 +522,7 @@ Status
SqliteMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFlag.throw_exception", throw std::exception());
//set all backup file to raw
ConnectorPtr->update_all(
@ -526,6 +544,7 @@ SqliteMetaImpl::UpdateTableFile(TableFileSchema& file_schema) {
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFile.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -554,6 +573,7 @@ Status
SqliteMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFiles.throw_exception", throw std::exception());
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -584,6 +604,7 @@ SqliteMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
}
return true;
});
fiu_do_on("SqliteMetaImpl.UpdateTableFiles.fail_commited", commited = false);
if (!commited) {
return HandleException("UpdateTableFiles error: sqlite transaction failed");
@ -600,6 +621,7 @@ Status
SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableIndex.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -659,6 +681,7 @@ Status
SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception", throw std::exception());
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -683,6 +706,7 @@ Status
SqliteMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DescribeTableIndex.throw_exception", throw std::exception());
auto groups = ConnectorPtr->select(columns(&TableSchema::engine_type_,
&TableSchema::nlist_,
@ -708,6 +732,7 @@ Status
SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.DropTableIndex.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -806,6 +831,7 @@ Status
SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.ShowPartitions.throw_exception", throw std::exception());
auto partitions = ConnectorPtr->select(columns(&TableSchema::table_id_),
where(c(&TableSchema::owner_table_) == table_id
@ -828,6 +854,7 @@ Status
SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) {
try {
server::MetricCollector metric;
fiu_do_on("SqliteMetaImpl.GetPartitionName.throw_exception", throw std::exception());
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
@ -859,6 +886,8 @@ SqliteMetaImpl::FilesToSearch(const std::string& table_id,
server::MetricCollector metric;
try {
fiu_do_on("SqliteMetaImpl.FilesToSearch.throw_exception", throw std::exception());
auto select_columns =
columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_,
&TableFileSchema::file_type_, &TableFileSchema::file_size_, &TableFileSchema::row_count_,
@ -972,6 +1001,8 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
files.clear();
try {
fiu_do_on("SqliteMetaImpl.FilesToMerge.throw_exception", throw std::exception());
server::MetricCollector metric;
// check table existence
@ -1040,6 +1071,8 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) {
files.clear();
try {
fiu_do_on("SqliteMetaImpl.FilesToIndex.throw_exception", throw std::exception());
server::MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
@ -1078,6 +1111,8 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) {
TableSchema table_schema;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
fiu_do_on("SqliteMetaImpl_FilesToIndex_TableNotFound",
status = Status(DB_NOT_FOUND, "table not found"));
if (!status.ok()) {
return status;
}
@ -1108,6 +1143,8 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
}
try {
fiu_do_on("SqliteMetaImpl.FilesByType.throw_exception", throw std::exception());
table_files.clear();
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::file_id_,
@ -1157,8 +1194,7 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
case (int)TableFileSchema::BACKUP:
++backup_count;
break;
default:
return Status(DB_ERROR, "Unknown file type.");
default:break;
}
table_files.emplace_back(file_schema);
@ -1167,29 +1203,25 @@ SqliteMetaImpl::FilesByType(const std::string& table_id,
std::string msg = "Get table files by type.";
for (int file_type : file_types) {
switch (file_type) {
case (int)TableFileSchema::RAW:
msg = msg + " raw files:" + std::to_string(raw_count);
case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count);
break;
case (int)TableFileSchema::NEW:
msg = msg + " new files:" + std::to_string(new_count);
case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count);
break;
case (int)TableFileSchema::NEW_MERGE:
msg = msg + " new_merge files:" + std::to_string(new_merge_count);
msg = msg + " new_merge files:"
+ std::to_string(new_merge_count);
break;
case (int)TableFileSchema::NEW_INDEX:
msg = msg + " new_index files:" + std::to_string(new_index_count);
msg = msg + " new_index files:"
+ std::to_string(new_index_count);
break;
case (int)TableFileSchema::TO_INDEX:
msg = msg + " to_index files:" + std::to_string(to_index_count);
case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count);
break;
case (int)TableFileSchema::INDEX:
msg = msg + " index files:" + std::to_string(index_count);
case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count);
break;
case (int)TableFileSchema::BACKUP:
msg = msg + " backup files:" + std::to_string(backup_count);
case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count);
break;
default:
return Status(DB_ERROR, "Unknown file type!");
default:break;
}
}
ENGINE_LOG_DEBUG << msg;
@ -1215,6 +1247,8 @@ SqliteMetaImpl::Archive() {
int64_t usecs = limit * DAY * US_PS;
int64_t now = utils::GetMicroSecTimeStamp();
try {
fiu_do_on("SqliteMetaImpl.Archive.throw_exception", throw std::exception());
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -1248,6 +1282,8 @@ Status
SqliteMetaImpl::Size(uint64_t& result) {
result = 0;
try {
fiu_do_on("SqliteMetaImpl.Size.throw_exception", throw std::exception());
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)),
where(c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE));
for (auto& total_size : selected) {
@ -1287,6 +1323,8 @@ SqliteMetaImpl::CleanUpShadowFiles() {
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpShadowFiles.fail_commited", commited = false);
fiu_do_on("SqliteMetaImpl.CleanUpShadowFiles.throw_exception", throw std::exception());
if (!commited) {
return HandleException("CleanUp error: sqlite transaction failed");
}
@ -1308,6 +1346,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove to_delete files
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException", throw std::exception());
server::MetricCollector metric;
std::vector<int> file_types = {
@ -1359,7 +1399,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// delete file from disk storage
utils::DeleteTableFilePath(options_, table_file);
ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:" << table_file.location_;
ENGINE_LOG_DEBUG << "Remove file id:" << table_file.file_id_ << " location:"
<< table_file.location_;
table_ids.insert(table_file.table_id_);
++clean_files;
@ -1367,6 +1408,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
}
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited", commited = false);
if (!commited) {
return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
@ -1381,6 +1423,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove to_delete tables
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException", throw std::exception());
server::MetricCollector metric;
// multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@ -1397,6 +1440,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
return true;
});
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited", commited = false);
if (!commited) {
return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
@ -1412,6 +1456,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
// remove deleted table folder
// don't remove table folder until all its files has been deleted
try {
fiu_do_on("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException", throw std::exception());
server::MetricCollector metric;
int64_t remove_tables = 0;
@ -1437,6 +1482,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) {
Status
SqliteMetaImpl::Count(const std::string& table_id, uint64_t& result) {
try {
fiu_do_on("SqliteMetaImpl.Count.throw_exception", throw std::exception());
server::MetricCollector metric;
std::vector<int> file_types = {(int)TableFileSchema::RAW, (int)TableFileSchema::TO_INDEX,
@ -1486,6 +1533,8 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) {
ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size;
try {
fiu_do_on("SqliteMetaImpl.DiscardFiles.throw_exception", throw std::exception());
server::MetricCollector metric;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@ -1525,7 +1574,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) {
return true;
});
fiu_do_on("SqliteMetaImpl.DiscardFiles.fail_commited", commited = false);
if (!commited) {
return HandleException("DiscardFiles error: sqlite transaction failed");
}

2
core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
View File

@ -16,6 +16,7 @@
// under the License.
#include <faiss/index_io.h>
#include <fiu-local.h>
#include <utility>
#include "knowhere/common/Exception.h"
@ -31,6 +32,7 @@ FaissBaseIndex::FaissBaseIndex(std::shared_ptr<faiss::Index> index) : index_(std
BinarySet
FaissBaseIndex::SerializeImpl() {
try {
fiu_do_on("FaissBaseIndex.SerializeImpl.throw_exception", throw std::exception());
faiss::Index* index = index_.get();
// SealImpl();

2
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp
View File

@ -21,6 +21,7 @@
#include <faiss/IndexFlat.h>
#include <faiss/MetaIndexes.h>
#include <faiss/index_io.h>
#include <fiu-local.h>
#ifdef MILVUS_GPU_VERSION
@ -61,6 +62,7 @@ GPUIDMAP::CopyGpuToCpu(const Config& config) {
BinarySet
GPUIDMAP::SerializeImpl() {
try {
fiu_do_on("GPUIDMP.SerializeImpl.throw_exception", throw std::exception());
MemoryIOWriter writer;
{
faiss::Index* index = index_.get();

6
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp
View File

@ -21,6 +21,7 @@
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/gpu/GpuIndexIVFFlat.h>
#include <faiss/index_io.h>
#include <fiu-local.h>
#include "knowhere/adapter/VectorAdapter.h"
#include "knowhere/common/Exception.h"
@ -81,6 +82,7 @@ GPUIVF::SerializeImpl() {
}
try {
fiu_do_on("GPUIVF.SerializeImpl.throw_exception", throw std::exception());
MemoryIOWriter writer;
{
faiss::Index* index = index_.get();
@ -128,7 +130,9 @@ void
GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
std::lock_guard<std::mutex> lk(mutex_);
if (auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_);
fiu_do_on("GPUIVF.search_impl.invald_index", device_index = nullptr);
if (device_index) {
auto search_cfg = std::dynamic_pointer_cast<IVFCfg>(cfg);
device_index->nprobe = search_cfg->nprobe;
// assert(device_index->getNumProbes() == search_cfg->nprobe);

3
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
View File

@ -29,6 +29,7 @@
#include <faiss/gpu/GpuCloner.h>
#endif
#include <fiu-local.h>
#include <chrono>
#include <memory>
#include <utility>
@ -119,6 +120,8 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
GETTENSOR(dataset)
try {
fiu_do_on("IVF.Search.throw_std_exception", throw std::exception());
fiu_do_on("IVF.Search.throw_faiss_exception", throw faiss::FaissException(""));
auto elems = rows * search_cfg->k;
size_t p_id_size = sizeof(int64_t) * elems;

6
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
View File

@ -21,11 +21,11 @@
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/helpers/FaissIO.h"
#include <utility>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/index_factory.h>
#include <fiu-local.h>
#include <utility>
namespace knowhere {
@ -300,7 +300,7 @@ IVFSQHybrid::SerializeImpl() {
if (!index_ || !index_->is_trained) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}
fiu_do_on("IVFSQHybrid.SerializeImpl.zero_gpu_mode", gpu_mode = 0);
if (gpu_mode == 0) {
MemoryIOWriter writer;
faiss::write_index(index_.get(), &writer);

6
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
View File

@ -19,12 +19,16 @@
#include "knowhere/adapter/VectorAdapter.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Timer.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include <fiu-local.h>
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexIVF.h"
#include "knowhere/index/vector_index/nsg/NSG.h"
@ -39,6 +43,7 @@ NSG::Serialize() {
}
try {
fiu_do_on("NSG.Serialize.throw_exception", throw std::exception());
algo::NsgIndex* index = index_.get();
MemoryIOWriter writer;
@ -57,6 +62,7 @@ NSG::Serialize() {
void
NSG::Load(const BinarySet& index_binary) {
try {
fiu_do_on("NSG.Load.throw_exception", throw std::exception());
auto binary = index_binary.GetByName("NSG");
MemoryIOReader reader;

2
core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
View File

@ -17,6 +17,7 @@
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include <fiu-local.h>
#include <utility>
namespace knowhere {
@ -83,6 +84,7 @@ FaissGpuResourceMgr::InitResource() {
ResPtr
FaissGpuResourceMgr::GetRes(const int64_t& device_id, const int64_t& alloc_size) {
fiu_return_on("FaissGpuResourceMgr.GetRes.ret_null", nullptr);
InitResource();
auto finder = idle_map_.find(device_id);

9
core/src/index/unittest/CMakeLists.txt
View File

@ -5,7 +5,7 @@ include_directories(${INDEX_SOURCE_DIR})
set(depend_libs
gtest gmock gtest_main gmock_main
faiss
faiss fiu
)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
@ -117,11 +117,18 @@ if (KNOWHERE_GPU_VERSION)
target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs})
endif ()
if (NOT TARGET test_knowhere_common)
add_executable(test_knowhere_common test_common.cpp ${util_srcs})
endif ()
target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf DESTINATION unittest)
install(TARGETS test_binaryivf DESTINATION unittest)
install(TARGETS test_idmap DESTINATION unittest)
install(TARGETS test_binaryidmap DESTINATION unittest)
install(TARGETS test_sptag DESTINATION unittest)
install(TARGETS test_knowhere_common DESTINATION unittest)
if (KNOWHERE_GPU_VERSION)
install(TARGETS test_gpuresource DESTINATION unittest)
install(TARGETS test_customized_index DESTINATION unittest)

51
core/src/index/unittest/test_common.cpp Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "knowhere/common/Dataset.h"
#include "knowhere/common/Timer.h"
#include "knowhere/knowhere/common/Exception.h"
#include "unittest/utils.h"
/*Some unittest for knowhere/common, mainly for improve code coverage.*/
TEST(COMMON_TEST, dataset_test) {
knowhere::Dataset set;
int64_t v1 = 111;
set.Set("key1", v1);
auto get_v1 = set.Get<int64_t>("key1");
ASSERT_EQ(get_v1, v1);
ASSERT_ANY_THROW(set.Get<int8_t>("key1"));
ASSERT_ANY_THROW(set.Get<int64_t>("dummy"));
}
TEST(COMMON_TEST, knowhere_exception) {
const std::string msg = "test";
knowhere::KnowhereException ex(msg);
ASSERT_EQ(ex.what(), msg);
}
TEST(COMMON_TEST, time_recoder) {
InitLog();
knowhere::TimeRecorder recoder("COMMTEST", 0);
sleep(1);
double span = recoder.ElapseFromBegin("get time");
ASSERT_GE(span, 1.0);
}

16
core/src/index/unittest/test_customized_index.cpp
View File

@ -15,8 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
#include <thread>
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "unittest/Helper.h"
#include "unittest/utils.h"
@ -55,6 +58,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
fiu_init(0);
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
@ -82,6 +86,7 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
auto cpu_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
cpu_idx->Load(binaryset);
ASSERT_ANY_THROW(cpu_idx->CopyCpuToGpuWithQuantizer(-1, conf));
auto pair = cpu_idx->CopyCpuToGpuWithQuantizer(DEVICEID, conf);
auto gpu_idx = pair.first;
auto quantization = pair.second;
@ -96,12 +101,21 @@ TEST_F(SingleIndexTest, IVFSQHybrid) {
for (int i = 0; i < 2; ++i) {
auto hybrid_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
hybrid_idx->Load(binaryset);
auto new_idx = hybrid_idx->LoadData(quantization, quantizer_conf);
auto result = new_idx->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
// PrintResult(result, nq, k);
}
{
// invalid quantizer config
quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();
auto hybrid_idx = std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, nullptr));
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf));
quantizer_conf->mode = 2; // only copy data
ASSERT_ANY_THROW(hybrid_idx->LoadData(quantization, quantizer_conf));
}
}
{

16
core/src/index/unittest/test_idmap.cpp
View File

@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
#include <iostream>
@ -54,6 +56,14 @@ TEST_F(IDMAPTest, idmap_basic) {
conf->k = k;
conf->metric_type = knowhere::METRICTYPE::L2;
// null faiss index
{
ASSERT_ANY_THROW(index_->Serialize());
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
ASSERT_ANY_THROW(index_->Add(nullptr, conf));
ASSERT_ANY_THROW(index_->AddWithoutId(nullptr, conf));
}
index_->Train(conf);
index_->Add(base_dataset, conf);
EXPECT_EQ(index_->Count(), nb);
@ -145,6 +155,7 @@ TEST_F(IDMAPTest, copy_test) {
{
// cpu to gpu
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, conf));
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, conf);
auto clone_result = clone_index->Search(query_dataset, conf);
AssertAnns(clone_result, nq, k);
@ -153,6 +164,11 @@ TEST_F(IDMAPTest, copy_test) {
ASSERT_THROW({ std::static_pointer_cast<knowhere::GPUIDMAP>(clone_index)->GetRawIds(); },
knowhere::KnowhereException);
fiu_init(0);
fiu_enable("GPUIDMP.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(clone_index->Serialize());
fiu_disable("GPUIDMP.SerializeImpl.throw_exception");
auto binary = clone_index->Serialize();
clone_index->Load(binary);
auto new_result = clone_index->Search(query_dataset, conf);

128
core/src/index/unittest/test_ivf.cpp
View File

@ -17,11 +17,15 @@
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <iostream>
#include <thread>
#ifdef MILVUS_GPU_VERSION
#include <faiss/gpu/GpuIndexIVFFlat.h>
#endif
#include "knowhere/adapter/VectorAdapter.h"
@ -33,11 +37,13 @@
#include "knowhere/index/vector_index/IndexIVFSQ.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include "unittest/Helper.h"
@ -54,15 +60,14 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM);
#endif
ParameterType parameter_type;
std::tie(index_type, parameter_type) = GetParam();
std::tie(index_type, parameter_type_) = GetParam();
// Init_with_default();
// nb = 1000000;
// nq = 1000;
// k = 1000;
Generate(DIM, NB, NQ);
index_ = IndexFactory(index_type);
conf = ParamGenerator::GetInstance().Gen(parameter_type);
conf = ParamGenerator::GetInstance().Gen(parameter_type_);
conf->Dump();
}
@ -77,6 +82,7 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P
std::string index_type;
knowhere::Config conf;
knowhere::IVFIndexPtr index_ = nullptr;
ParameterType parameter_type_;
};
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
@ -98,6 +104,10 @@ TEST_P(IVFTest, ivf_basic) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
// null faiss index
ASSERT_ANY_THROW(index_->Add(base_dataset, conf));
ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, conf));
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
@ -107,9 +117,14 @@ TEST_P(IVFTest, ivf_basic) {
auto result = index_->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
// PrintResult(result, nq, k);
#ifdef MILVUS_GPU_VERSION
knowhere::FaissGpuResourceMgr::GetInstance().Dump();
#endif
}
TEST_P(IVFTest, ivf_serialize) {
fiu_init(0);
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void*>(bin->data.get()), bin->size);
@ -118,12 +133,23 @@ TEST_P(IVFTest, ivf_serialize) {
reader(ret, bin->size);
};
{
// null faisss index serialize
ASSERT_ANY_THROW(index_->Serialize());
knowhere::IVFIndexModel model(nullptr);
ASSERT_ANY_THROW(model.Serialize());
}
{
// serialize index-model
auto model = index_->Train(base_dataset, conf);
auto binaryset = model->Serialize();
auto bin = binaryset.GetByName("IVF");
fiu_enable("FaissBaseIndex.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(model->Serialize());
fiu_disable("FaissBaseIndex.SerializeImpl.throw_exception");
std::string filename = "/tmp/ivf_test_model_serialize.bin";
auto load_data = new uint8_t[bin->size];
serialize(filename, bin, load_data);
@ -250,6 +276,7 @@ TEST_P(IVFTest, clone_test) {
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
EXPECT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, knowhere::Config()));
} else {
EXPECT_THROW(
{
@ -260,6 +287,7 @@ TEST_P(IVFTest, clone_test) {
}
}
}
#endif
#ifdef MILVUS_GPU_VERSION
@ -276,6 +304,9 @@ TEST_P(IVFTest, gpu_seal_test) {
auto preprocessor = index_->BuildPreprocessor(base_dataset, conf);
index_->set_preprocessor(preprocessor);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
ASSERT_ANY_THROW(index_->Seal());
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
index_->Add(base_dataset, conf);
@ -284,6 +315,14 @@ TEST_P(IVFTest, gpu_seal_test) {
auto result = index_->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
fiu_init(0);
fiu_enable("IVF.Search.throw_std_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
fiu_disable("IVF.Search.throw_std_exception");
fiu_enable("IVF.Search.throw_faiss_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(query_dataset, conf));
fiu_disable("IVF.Search.throw_faiss_exception");
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
knowhere::TimeRecorder tc("CopyToGpu");
@ -294,6 +333,89 @@ TEST_P(IVFTest, gpu_seal_test) {
knowhere::cloner::CopyCpuToGpu(cpu_idx, DEVICEID, knowhere::Config());
auto with_seal = tc.RecordSection("With seal");
ASSERT_GE(without_seal, with_seal);
// copy to GPU with invalid device id
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(cpu_idx, -1, knowhere::Config()));
}
#endif
TEST_P(IVFTest, invalid_gpu_source) {
std::vector<std::string> support_idx_vec{"GPUIVF", "GPUIVFPQ", "GPUIVFSQ"};
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder == support_idx_vec.cend()) {
return;
}
auto invalid_conf = ParamGenerator::GetInstance().Gen(parameter_type_);
invalid_conf->gpu_id = -1;
if (index_type == "GPUIVF") {
// null faiss index
knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
}
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
auto binaryset = model->Serialize();
fiu_init(0);
fiu_enable("GPUIVF.SerializeImpl.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Serialize());
fiu_disable("GPUIVF.SerializeImpl.throw_exception");
fiu_enable("GPUIVF.search_impl.invald_index", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Search(base_dataset, invalid_conf));
fiu_disable("GPUIVF.search_impl.invald_index");
auto ivf_index = std::dynamic_pointer_cast<knowhere::GPUIVF>(index_);
if (ivf_index) {
auto gpu_index = std::dynamic_pointer_cast<knowhere::GPUIndex>(ivf_index);
gpu_index->SetGpuDevice(-1);
ASSERT_EQ(gpu_index->GetGpuDevice(), -1);
ASSERT_ANY_THROW(index_->set_index_model(model));
}
ASSERT_ANY_THROW(index_->Load(binaryset));
ASSERT_ANY_THROW(index_->Train(base_dataset, invalid_conf));
}
#ifdef CUSTOMIZATION
TEST_P(IVFTest, IVFSQHybrid_test) {
std::vector<std::string> support_idx_vec{"IVFSQHybrid"};
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder == support_idx_vec.cend()) {
return;
}
fiu_init(0);
knowhere::cloner::CopyGpuToCpu(index_, conf);
ASSERT_ANY_THROW(knowhere::cloner::CopyCpuToGpu(index_, -1, conf));
fiu_enable("FaissGpuResourceMgr.GetRes.ret_null", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Train(base_dataset, conf));
ASSERT_ANY_THROW(index_->CopyCpuToGpu(DEVICEID, conf));
fiu_disable("FaissGpuResourceMgr.GetRes.ret_null");
auto model = index_->Train(base_dataset, conf);
index_->set_index_model(model);
auto index = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_);
ASSERT_TRUE(index != nullptr);
ASSERT_ANY_THROW(index->UnsetQuantizer());
knowhere::QuantizerConfig config = std::make_shared<knowhere::QuantizerCfg>();
config->gpu_id = knowhere::INVALID_VALUE;
// mode = -1
ASSERT_ANY_THROW(index->LoadQuantizer(config));
config->mode = 1;
ASSERT_ANY_THROW(index->LoadQuantizer(config));
config->gpu_id = DEVICEID;
// index->LoadQuantizer(config);
ASSERT_ANY_THROW(index->SetQuantizer(nullptr));
}
#endif
#endif

26
core/src/index/unittest/test_nsg/test_nsg.cpp
View File

@ -30,6 +30,8 @@
#include "knowhere/common/Timer.h"
#include "knowhere/index/vector_index/nsg/NSGIO.h"
#include <fiu-control.h>
#include <fiu-local.h>
#include "unittest/utils.h"
using ::testing::Combine;
@ -42,7 +44,7 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
protected:
void
SetUp() override {
// Init_with_default();
// Init_with_default();
#ifdef MILVUS_GPU_VERSION
int64_t MB = 1024 * 1024;
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, MB * 200, MB * 600, 1);
@ -85,14 +87,34 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
TEST_F(NSGInterfaceTest, basic_test) {
assert(!xb.empty());
fiu_init(0);
// untrained index
{
ASSERT_ANY_THROW(index_->Search(query_dataset, search_conf));
ASSERT_ANY_THROW(index_->Serialize());
}
train_conf->gpu_id = knowhere::INVALID_VALUE;
auto model_invalid_gpu = index_->Train(base_dataset, train_conf);
train_conf->gpu_id = DEVICEID;
auto model = index_->Train(base_dataset, train_conf);
auto result = index_->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
auto binaryset = index_->Serialize();
{
fiu_enable("NSG.Serialize.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(index_->Serialize());
fiu_disable("NSG.Serialize.throw_exception");
}
auto new_index = std::make_shared<knowhere::NSG>();
new_index->Load(binaryset);
{
fiu_enable("NSG.Load.throw_exception", 1, nullptr, 0);
ASSERT_ANY_THROW(new_index->Load(binaryset));
fiu_disable("NSG.Load.throw_exception");
}
auto new_result = new_index->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);

10
core/src/index/unittest/test_sptag.cpp
View File

@ -95,6 +95,16 @@ TEST_P(SPTAGTest, sptag_basic) {
std::cout << "id\n" << ss_id.str() << std::endl;
std::cout << "dist\n" << ss_dist.str() << std::endl;
}
// Though these functions do nothing, use them to improve code coverage
{
index_->Seal();
knowhere::CPUSPTAGRNGIndexModel index_model;
// Function Serialize's implementation do'nt have return value,
// which will cause undefined behavior.
// index_model.Serialize();
index_model.Load(knowhere::BinarySet());
}
}
TEST_P(SPTAGTest, sptag_serialize) {

6
core/src/index/unittest/utils.cpp
View File

@ -72,6 +72,8 @@ BinaryDataGen::Generate(const int& dim, const int& nb, const int& nq) {
query_dataset = generate_binary_query_dataset(nq, dim, xq.data());
}
// not used
#if 0
knowhere::DatasetPtr
DataGen::GenQuery(const int& nq) {
xq.resize(nq * dim);
@ -80,6 +82,7 @@ DataGen::GenQuery(const int& nq) {
}
return generate_query_dataset(nq, dim, xq.data());
}
#endif
void
GenAll(const int64_t dim, const int64_t& nb, std::vector<float>& xb, std::vector<int64_t>& ids, const int64_t& nq,
@ -236,6 +239,8 @@ PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
std::cout << "dist\n" << ss_dist.str() << std::endl;
}
// not used
#if 0
void
Load_nns_graph(std::vector<std::vector<int64_t>>& final_graph, const char* filename) {
std::vector<std::vector<unsigned>> knng;
@ -304,3 +309,4 @@ int* // not very clean, but works as long as sizeof(int) == sizeof(float)
ivecs_read(const char* fname, size_t* d_out, size_t* n_out) {
return (int*)fvecs_read(fname, d_out, n_out);
}
#endif

16
core/src/metrics/SystemInfo.cpp
View File

@ -20,13 +20,16 @@
#include "utils/Log.h"
#include <dirent.h>
#include <fiu-local.h>
#include <sys/sysinfo.h>
#include <sys/times.h>
#include <unistd.h>
#include <map>
#ifdef MILVUS_GPU_VERSION
#include <nvml.h>
#endif
namespace milvus {
@ -64,11 +67,13 @@ SystemInfo::Init() {
// initialize GPU information
nvmlReturn_t nvmlresult;
nvmlresult = nvmlInit();
fiu_do_on("SystemInfo.Init.nvmInit_fail", nvmlresult = NVML_ERROR_NOT_FOUND);
if (NVML_SUCCESS != nvmlresult) {
SERVER_LOG_ERROR << "System information initilization failed";
return;
}
nvmlresult = nvmlDeviceGetCount(&num_device_);
fiu_do_on("SystemInfo.Init.nvm_getDevice_fail", nvmlresult = NVML_ERROR_NOT_FOUND);
if (NVML_SUCCESS != nvmlresult) {
SERVER_LOG_ERROR << "Unable to get devidce number";
return;
@ -127,6 +132,7 @@ SystemInfo::GetProcessUsedMemory() {
double
SystemInfo::MemoryPercent() {
fiu_do_on("SystemInfo.MemoryPercent.mock", initialized_ = false);
if (!initialized_) {
Init();
}
@ -144,7 +150,7 @@ SystemInfo::CPUCorePercent() {
std::vector<uint64_t> cur_total_time_array = getTotalCpuTime(cur_work_time_array);
std::vector<double> cpu_core_percent;
for (int i = 1; i < num_processors_; i++) {
for (int i = 0; i < cur_total_time_array.size(); i++) {
double total_cpu_time = cur_total_time_array[i] - prev_total_time_array[i];
double cpu_work_time = cur_work_time_array[i] - prev_work_time_array[i];
cpu_core_percent.push_back((cpu_work_time / total_cpu_time) * 100);
@ -156,6 +162,7 @@ std::vector<uint64_t>
SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
std::vector<uint64_t> total_time_array;
FILE* file = fopen("/proc/stat", "r");
fiu_do_on("SystemInfo.getTotalCpuTime.open_proc", file = NULL);
if (file == NULL) {
SERVER_LOG_ERROR << "Could not open stat file";
return total_time_array;
@ -167,6 +174,7 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
for (int i = 0; i < num_processors_; i++) {
char buffer[1024];
char* ret = fgets(buffer, sizeof(buffer) - 1, file);
fiu_do_on("SystemInfo.getTotalCpuTime.read_proc", ret = NULL);
if (ret == NULL) {
SERVER_LOG_ERROR << "Could not read stat file";
fclose(file);
@ -186,6 +194,7 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
double
SystemInfo::CPUPercent() {
fiu_do_on("SystemInfo.CPUPercent.mock", initialized_ = false);
if (!initialized_) {
Init();
}
@ -212,6 +221,7 @@ SystemInfo::CPUPercent() {
std::vector<uint64_t>
SystemInfo::GPUMemoryTotal() {
// get GPU usage percent
fiu_do_on("SystemInfo.GPUMemoryTotal.mock", initialized_ = false);
if (!initialized_)
Init();
std::vector<uint64_t> result;
@ -232,6 +242,7 @@ SystemInfo::GPUMemoryTotal() {
std::vector<uint64_t>
SystemInfo::GPUTemperature() {
fiu_do_on("SystemInfo.GPUTemperature.mock", initialized_ = false);
if (!initialized_)
Init();
std::vector<uint64_t> result;
@ -258,6 +269,7 @@ SystemInfo::CPUTemperature() {
DIR* dir = NULL;
dir = opendir(path.c_str());
fiu_do_on("SystemInfo.CPUTemperature.opendir", dir = NULL);
if (!dir) {
SERVER_LOG_ERROR << "Could not open hwmon directory";
return result;
@ -275,6 +287,7 @@ SystemInfo::CPUTemperature() {
std::string object = filename;
object += "/temp1_input";
FILE* file = fopen(object.c_str(), "r");
fiu_do_on("SystemInfo.CPUTemperature.openfile", file = NULL);
if (file == nullptr) {
SERVER_LOG_ERROR << "Could not open temperature file";
return result;
@ -292,6 +305,7 @@ SystemInfo::CPUTemperature() {
std::vector<uint64_t>
SystemInfo::GPUMemoryUsed() {
// get GPU memory used
fiu_do_on("SystemInfo.GPUMemoryUsed.mock", initialized_ = false);
if (!initialized_)
Init();

4
core/src/scheduler/SchedInst.cpp
View File

@ -20,6 +20,7 @@
#include "Utils.h"
#include "server/Config.h"
#include <fiu-local.h>
#include <set>
#include <string>
#include <utility>
@ -52,7 +53,7 @@ load_simple_config() {
ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0));
ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
// get resources
// get resources
#ifdef MILVUS_GPU_VERSION
bool enable_gpu = false;
server::Config& config = server::Config::GetInstance();
@ -63,6 +64,7 @@ load_simple_config() {
std::vector<int64_t> build_gpu_ids;
config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids);
auto pcie = Connection("pcie", 12000);
fiu_do_on("load_simple_config_mock", build_gpu_ids.push_back(1));
std::vector<int64_t> not_find_build_ids;
for (auto& build_id : build_gpu_ids) {

2
core/src/scheduler/TaskTable.cpp
View File

@ -283,7 +283,7 @@ TaskTable::TaskToExecute() {
auto begin = table_.front() + 1;
for (size_t i = 0; i < table_.size(); ++i) {
auto index = begin + i;
if (table_[index]->state == TaskTableItemState::LOADED) {
if (table_[index] && table_[index]->state == TaskTableItemState::LOADED) {
++count;
}
}

5
core/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -42,6 +42,8 @@ get_neighbours(const ResourcePtr& self) {
return neighbours;
}
// This function has not been invoked, comment it for code coverage
#if 0
std::vector<std::pair<ResourcePtr, Connection>>
get_neighbours_with_connetion(const ResourcePtr& self) {
std::vector<std::pair<ResourcePtr, Connection>> neighbours;
@ -87,6 +89,7 @@ Action::PushTaskToNeighbourRandomly(TaskTableItemPtr task_item, const ResourcePt
// TODO(wxyu): process
}
}
#endif
void
Action::PushTaskToAllNeighbour(TaskTableItemPtr task_item, const ResourcePtr& self) {
@ -96,10 +99,12 @@ Action::PushTaskToAllNeighbour(TaskTableItemPtr task_item, const ResourcePtr& se
}
}
#if 0
void
Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) {
dest->task_table().Put(task_item->task, task_item);
}
#endif
void
Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource,

8
core/src/scheduler/optimizer/BuildIndexPass.cpp
View File

@ -14,10 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#ifdef MILVUS_GPU_VERSION
namespace milvus {
@ -27,8 +28,9 @@ void
BuildIndexPass::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_);
fiu_do_on("BuildIndexPass.Init.get_config_fail", s = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!s.ok()) {
throw;
throw std::exception();
}
}
@ -36,7 +38,7 @@ bool
BuildIndexPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::BuildIndexTask)
return false;
fiu_do_on("BuildIndexPass.Run.empty_gpu_ids", build_gpu_ids_.clear());
if (build_gpu_ids_.empty()) {
SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!";
return false;

3
core/src/scheduler/optimizer/FaissFlatPass.cpp
View File

@ -24,6 +24,7 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
namespace milvus {
namespace scheduler {
@ -36,7 +37,7 @@ FaissFlatPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
}

2
core/src/scheduler/optimizer/FaissIVFFlatPass.cpp
View File

@ -37,7 +37,7 @@ FaissIVFFlatPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}

4
core/src/scheduler/optimizer/FaissIVFPQPass.cpp
View File

@ -24,6 +24,8 @@
#include "server/Config.h"
#include "utils/Log.h"
#include <fiu-local.h>
namespace milvus {
namespace scheduler {
@ -37,7 +39,7 @@ FaissIVFPQPass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}

2
core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp
View File

@ -37,7 +37,7 @@ FaissIVFSQ8Pass::Init() {
}
s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) {
throw;
throw std::exception();
}
#endif
}

13
core/src/scheduler/task/BuildIndexTask.cpp
View File

@ -23,6 +23,7 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <thread>
@ -60,12 +61,13 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
error_msg = "Wrong load type";
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XBuildIndexTask.Load.throw_std_exception", throw std::exception());
} catch (std::exception& ex) {
// typical error: out of disk space or permition denied
error_msg = "Failed to load to_index file: " + std::string(ex.what());
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XBuildIndexTask.Load.out_of_memory", stat = Status(SERVER_UNEXPECTED_ERROR, "out of memory"));
if (!stat.ok()) {
Status s;
if (stat.ToString().find("out of memory") != std::string::npos) {
@ -117,7 +119,8 @@ XBuildIndexTask::Execute() {
table_file.file_type_ = engine::meta::TableFileSchema::NEW_INDEX;
engine::meta::MetaPtr meta_ptr = build_index_job->meta();
Status status = build_index_job->meta()->CreateTableFile(table_file);
Status status = meta_ptr->CreateTableFile(table_file);
fiu_do_on("XBuildIndexTask.Execute.create_table_success", status = Status::OK());
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
build_index_job->BuildIndexDone(to_index_id_);
@ -130,6 +133,7 @@ XBuildIndexTask::Execute() {
try {
ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_;
index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
fiu_do_on("XBuildIndexTask.Execute.build_index_fail", index = nullptr);
if (index == nullptr) {
throw Exception(DB_ERROR, "index NULL");
}
@ -150,6 +154,8 @@ XBuildIndexTask::Execute() {
// step 4: if table has been deleted, dont save index file
bool has_table = false;
meta_ptr->HasTable(file_->table_id_, has_table);
fiu_do_on("XBuildIndexTask.Execute.has_table", has_table = true);
if (!has_table) {
meta_ptr->DeleteTableFiles(file_->table_id_);
@ -161,6 +167,7 @@ XBuildIndexTask::Execute() {
// step 5: save index file
try {
fiu_do_on("XBuildIndexTask.Execute.throw_std_exception", throw std::exception());
status = index->Serialize();
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
@ -171,6 +178,7 @@ XBuildIndexTask::Execute() {
status = Status(DB_ERROR, msg);
}
fiu_do_on("XBuildIndexTask.Execute.save_index_file_success", status = Status::OK());
if (!status.ok()) {
// if failed to serialize index file to disk
// typical error: out of disk space, out of memory or permition denied
@ -201,6 +209,7 @@ XBuildIndexTask::Execute() {
status = meta_ptr->UpdateTableFiles(update_files);
}
fiu_do_on("XBuildIndexTask.Execute.update_table_file_fail", status = Status(SERVER_UNEXPECTED_ERROR, ""));
if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize()
<< " bytes"

2
core/src/scheduler/task/Path.h
View File

@ -64,7 +64,7 @@ class Path {
if (!path_.empty()) {
return path_[0];
} else {
return nullptr;
return "";
}
}

12
core/src/scheduler/task/SearchTask.cpp
View File

@ -15,10 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/task/SearchTask.h"
#include <src/scheduler/SchedInst.h>
#include <fiu-local.h>
#include <algorithm>
#include <memory>
#include <string>
@ -27,7 +24,9 @@
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/SchedInst.h"
#include "scheduler/job/SearchJob.h"
#include "scheduler/task/SearchTask.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
@ -123,6 +122,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
std::string type_str;
try {
fiu_do_on("XSearchTask.Load.throw_std_exception", throw std::exception());
if (type == LoadType::DISK2CPU) {
stat = index_engine_->Load();
type_str = "DISK2CPU";
@ -145,6 +145,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
error_msg = "Failed to load index file: " + std::string(ex.what());
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
fiu_do_on("XSearchTask.Load.out_of_memory", stat = Status(SERVER_UNEXPECTED_ERROR, "out of memory"));
if (!stat.ok()) {
Status s;
@ -217,6 +218,7 @@ XSearchTask::Execute() {
"job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk);
try {
fiu_do_on("XSearchTask.Execute.throw_std_exception", throw std::exception());
// step 2: search
bool hybrid = false;
if (index_engine_->IndexEngineType() == engine::EngineType::FAISS_IVFSQ8H &&
@ -231,6 +233,8 @@ XSearchTask::Execute() {
s = index_engine_->Search(nq, vectors.binary_data_.data(), topk, nprobe, output_distance.data(),
output_ids.data(), hybrid);
}
fiu_do_on("XSearchTask.Execute.search_fail", s = Status(SERVER_UNEXPECTED_ERROR, ""));
if (!s.ok()) {
search_job->GetStatus() = s;
search_job->SearchDone(index_id_);

97
core/src/server/Config.cpp
View File

@ -32,6 +32,8 @@
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
namespace milvus {
namespace server {
@ -57,15 +59,10 @@ Config::LoadConfigFile(const std::string& filename) {
return Status(SERVER_FILE_NOT_FOUND, str);
}
try {
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
Status s = mgr->LoadConfigFile(filename);
if (!s.ok()) {
return s;
}
} catch (YAML::Exception& e) {
std::string str = "Exception occurs when loading config file: " + filename;
return Status(SERVER_UNEXPECTED_ERROR, str);
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
Status s = mgr->LoadConfigFile(filename);
if (!s.ok()) {
return s;
}
return Status::OK();
@ -349,7 +346,9 @@ Config::ProcessConfigCli(std::string& result, const std::string& cmd) {
////////////////////////////////////////////////////////////////////////////////
Status
Config::CheckConfigVersion(const std::string& value) {
if (milvus_config_version_map.at(MILVUS_VERSION) != value) {
bool exist_error = milvus_config_version_map.at(MILVUS_VERSION) != value;
fiu_do_on("check_config_version_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid config version: " + value +
". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION);
return Status(SERVER_INVALID_ARGUMENT, msg);
@ -360,7 +359,10 @@ Config::CheckConfigVersion(const std::string& value) {
/* server config */
Status
Config::CheckServerConfigAddress(const std::string& value) {
if (!ValidationUtil::ValidateIpAddress(value).ok()) {
auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok();
fiu_do_on("check_config_address_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
@ -370,7 +372,10 @@ Config::CheckServerConfigAddress(const std::string& value) {
Status
Config::CheckServerConfigPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_port_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
@ -386,6 +391,10 @@ Config::CheckServerConfigPort(const std::string& value) {
Status
Config::CheckServerConfigDeployMode(const std::string& value) {
fiu_return_on("check_config_deploy_mode_fail",
Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."));
if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") {
return Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.");
@ -395,6 +404,9 @@ Config::CheckServerConfigDeployMode(const std::string& value) {
Status
Config::CheckServerConfigTimeZone(const std::string& value) {
fiu_return_on("check_config_time_zone_fail",
Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value));
if (value.length() <= 3) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value);
} else {
@ -431,7 +443,10 @@ Config::CheckServerConfigWebPort(const std::string& value) {
/* DB config */
Status
Config::CheckDBConfigBackendUrl(const std::string& value) {
if (!ValidationUtil::ValidateDbURI(value).ok()) {
auto exist_error = !ValidationUtil::ValidateDbURI(value).ok();
fiu_do_on("check_config_backend_url_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " +
"The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus.";
@ -442,7 +457,10 @@ Config::CheckDBConfigBackendUrl(const std::string& value) {
Status
Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_disk_threshold_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid archive disk threshold: " + value +
". Possible reason: db_config.archive_disk_threshold is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
@ -452,7 +470,10 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
Status
Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
fiu_do_on("check_config_archive_days_threshold_fail", exist_error = true);
if (exist_error) {
std::string msg = "Invalid archive days threshold: " + value +
". Possible reason: db_config.archive_days_threshold is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
@ -463,6 +484,7 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
/* storage config */
Status
Config::CheckStorageConfigPrimaryPath(const std::string& value) {
fiu_return_on("check_config_primary_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.db_path is empty.");
}
@ -471,6 +493,7 @@ Config::CheckStorageConfigPrimaryPath(const std::string& value) {
Status
Config::CheckStorageConfigSecondaryPath(const std::string& value) {
fiu_return_on("check_config_secondary_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
return Status::OK();
}
@ -536,7 +559,10 @@ Config::CheckStorageConfigS3Bucket(const std::string& value) {
/* metric config */
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
fiu_do_on("check_config_enable_monitor_fail", exist_error = true);
if (exist_error) {
std::string msg =
"Invalid metric config: " + value + ". Possible reason: metric_config.enable_monitor is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
@ -572,6 +598,8 @@ Config::CheckMetricConfigPort(const std::string& value) {
/* cache config */
Status
Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
fiu_return_on("check_config_cpu_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid cpu cache capacity: " + value +
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
@ -598,6 +626,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
CONFIG_CHECK(GetCacheConfigInsertBufferSize(buffer_value));
int64_t insert_buffer_size = buffer_value * GB;
fiu_do_on("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer", insert_buffer_size = total_mem + 1);
if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
std::string msg = "Invalid cpu cache capacity: " + value +
". Possible reason: sum of cache_config.cpu_cache_capacity and "
@ -610,6 +639,8 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
Status
Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
fiu_return_on("check_config_cpu_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid cpu cache threshold: " + value +
". Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0].";
@ -627,6 +658,7 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
Status
Config::CheckCacheConfigInsertBufferSize(const std::string& value) {
fiu_return_on("check_config_insert_buffer_size_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid insert buffer size: " + value +
". Possible reason: cache_config.insert_buffer_size is not a positive integer.";
@ -652,6 +684,8 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) {
Status
Config::CheckCacheConfigCacheInsertData(const std::string& value) {
fiu_return_on("check_config_cache_insert_data_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid cache insert data option: " + value +
". Possible reason: cache_config.cache_insert_data is not a boolean.";
@ -663,6 +697,8 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) {
/* engine config */
Status
Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
fiu_return_on("check_config_use_blas_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid use blas threshold: " + value +
". Possible reason: engine_config.use_blas_threshold is not a positive integer.";
@ -673,6 +709,8 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
Status
Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
fiu_return_on("check_config_omp_thread_num_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid omp thread num: " + value +
". Possible reason: engine_config.omp_thread_num is not a positive integer.";
@ -694,6 +732,8 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
Status
Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_search_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu search threshold: " + value +
". Possible reason: engine_config.gpu_search_threshold is not a positive integer.";
@ -705,6 +745,8 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) {
/* gpu resource config */
Status
Config::CheckGpuResourceConfigEnable(const std::string& value) {
fiu_return_on("check_config_gpu_resource_enable_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg =
"Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean.";
@ -715,6 +757,8 @@ Config::CheckGpuResourceConfigEnable(const std::string& value) {
Status
Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
fiu_return_on("check_gpu_resource_config_cache_capacity_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu cache capacity: " + value +
". Possible reason: gpu_resource_config.cache_capacity is not a positive integer.";
@ -743,6 +787,8 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
Status
Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) {
fiu_return_on("check_config_gpu_resource_cache_threshold_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid gpu cache threshold: " + value +
". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0].";
@ -785,6 +831,8 @@ CheckGpuResource(const std::string& value) {
Status
Config::CheckGpuResourceConfigSearchResources(const std::vector<std::string>& value) {
fiu_return_on("check_gpu_resource_config_search_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
std::string msg =
"Invalid gpu search resource. "
@ -810,6 +858,8 @@ Config::CheckGpuResourceConfigSearchResources(const std::vector<std::string>& va
Status
Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>& value) {
fiu_return_on("check_gpu_resource_config_build_index_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty()) {
std::string msg =
"Invalid gpu build index resource. "
@ -852,10 +902,7 @@ Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_
if (config_map_.find(parent_key) == config_map_.end()) {
return false;
}
if (config_map_[parent_key].count(child_key) == 0) {
return false;
}
return true;
return config_map_[parent_key].count(child_key) != 0;
}
Status
@ -1111,10 +1158,12 @@ Config::GetEngineConfigGpuSearchThreshold(int64_t& value) {
value = std::stoll(str);
return Status::OK();
}
#endif
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
Status
Config::GetGpuResourceConfigEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT);
@ -1128,6 +1177,7 @@ Status
Config::GetGpuResourceConfigCacheCapacity(int64_t& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
@ -1143,6 +1193,7 @@ Status
Config::GetGpuResourceConfigCacheThreshold(float& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
@ -1158,6 +1209,7 @@ Status
Config::GetGpuResourceConfigSearchResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("get_gpu_config_search_resources.disable_gpu_resource_fail", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
@ -1177,6 +1229,7 @@ Status
Config::GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false;
CONFIG_CHECK(GetGpuResourceConfigEnable(gpu_resource_enable));
fiu_do_on("get_gpu_config_build_index_resources.disable_gpu_resource_fail", gpu_resource_enable = false);
if (!gpu_resource_enable) {
std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false.";
return Status(SERVER_UNSUPPORTED_ERROR, msg);
@ -1199,6 +1252,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value) {
Status
Config::GetTracingConfigJsonConfigPath(std::string& value) {
value = GetConfigStr(CONFIG_TRACING, CONFIG_TRACING_JSON_CONFIG_PATH, "");
fiu_do_on("get_config_json_config_path_fail", value = "error_config_json_path");
if (!value.empty()) {
std::ifstream tracer_config(value);
Status s = tracer_config.good() ? Status::OK()
@ -1374,10 +1428,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value));
return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
}
#endif
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
Status
Config::SetGpuResourceConfigEnable(const std::string& value) {
CONFIG_CHECK(CheckGpuResourceConfigEnable(value));
@ -1411,6 +1467,7 @@ Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) {
CONFIG_CHECK(CheckGpuResourceConfigBuildIndexResources(res_vec));
return SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value);
}
#endif
} // namespace server

12
core/src/server/delivery/RequestScheduler.cpp
View File

@ -18,6 +18,8 @@
#include "server/delivery/RequestScheduler.h"
#include "utils/Log.h"
#include <fiu-local.h>
#include <unistd.h>
#include <utility>
namespace milvus {
@ -74,7 +76,6 @@ RequestScheduler::Stop() {
for (auto& iter : execute_threads_) {
if (iter == nullptr)
continue;
iter->join();
}
request_groups_.clear();
@ -90,6 +91,8 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) {
}
auto status = PutToQueue(request_ptr);
fiu_do_on("RequestScheduler.ExecuteRequest.push_queue_fail", status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!status.ok()) {
SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString();
return status;
@ -98,7 +101,6 @@ RequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) {
if (request_ptr->IsAsync()) {
return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere
}
return request_ptr->WaitToFinish(); // sync execution
}
@ -116,7 +118,10 @@ RequestScheduler::TakeToExecute(RequestQueuePtr request_queue) {
}
try {
fiu_do_on("RequestScheduler.TakeToExecute.throw_std_exception1", throw std::exception());
auto status = request->Execute();
fiu_do_on("RequestScheduler.TakeToExecute.throw_std_exception", throw std::exception());
fiu_do_on("RequestScheduler.TakeToExecute.execute_fail", status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!status.ok()) {
SERVER_LOG_ERROR << "Request failed with code: " << status.ToString();
}
@ -137,9 +142,12 @@ RequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) {
RequestQueuePtr queue = std::make_shared<RequestQueue>();
queue->Put(request_ptr);
request_groups_.insert(std::make_pair(group_name, queue));
fiu_do_on("RequestScheduler.PutToQueue.null_queue", queue = nullptr);
// start a thread
ThreadPtr thread = std::make_shared<std::thread>(&RequestScheduler::TakeToExecute, this, queue);
fiu_do_on("RequestScheduler.PutToQueue.push_null_thread", execute_threads_.push_back(nullptr));
execute_threads_.push_back(thread);
SERVER_LOG_INFO << "Create new thread for request group: " << group_name;
}

6
core/src/server/delivery/request/CountTableRequest.cpp
View File

@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -52,8 +53,11 @@ CountTableRequest::OnExecute() {
// step 2: get row count
uint64_t row_count = 0;
status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
fiu_do_on("CountTableRequest.OnExecute.db_not_found", status = Status(DB_NOT_FOUND, ""));
fiu_do_on("CountTableRequest.OnExecute.status_error", status = Status(SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CountTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code(), DB_NOT_FOUND) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
} else {
return status;

8
core/src/server/delivery/request/CreateIndexRequest.cpp
View File

@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
@ -53,6 +54,8 @@ CreateIndexRequest::OnExecute() {
bool has_table = false;
status = DBWrapper::DB()->HasTable(table_name_, has_table);
fiu_do_on("CreateIndexRequest.OnExecute.not_has_table", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreateIndexRequest.OnExecute.throw_std.exception", throw std::exception());
if (!status.ok()) {
return status;
}
@ -92,6 +95,9 @@ CreateIndexRequest::OnExecute() {
bool enable_gpu = false;
server::Config& config = server::Config::GetInstance();
s = config.GetGpuResourceConfigEnable(enable_gpu);
fiu_do_on("CreateIndexRequest.OnExecute.ip_meteric",
table_info.metric_type_ = static_cast<int>(engine::MetricType::IP));
if (s.ok() && adapter_index_type == (int)engine::EngineType::FAISS_PQ &&
table_info.metric_type_ == (int)engine::MetricType::IP) {
return Status(SERVER_UNEXPECTED_ERROR, "PQ not support IP in GPU version!");
@ -103,6 +109,8 @@ CreateIndexRequest::OnExecute() {
index.engine_type_ = adapter_index_type;
index.nlist_ = nlist_;
status = DBWrapper::DB()->CreateIndex(table_name_, index);
fiu_do_on("CreateIndexRequest.OnExecute.create_index_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

11
core/src/server/delivery/request/CreatePartitionRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
@ -47,22 +48,32 @@ CreatePartitionRequest::OnExecute() {
try {
// step 1: check arguments
auto status = ValidationUtil::ValidateTableName(table_name_);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionName(partition_name_);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({tag_});
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
// step 2: create partition
status = DBWrapper::DB()->CreatePartition(table_name_, partition_name_, tag_);
fiu_do_on("CreatePartitionRequest.OnExecute.db_already_exist", status = Status(milvus::DB_ALREADY_EXIST, ""));
fiu_do_on("CreatePartitionRequest.OnExecute.create_partition_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreatePartitionRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
// partition could exist
if (status.code() == DB_ALREADY_EXIST) {

7
core/src/server/delivery/request/CreateTableRequest.cpp
View File

@ -22,6 +22,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
@ -62,6 +63,8 @@ CreateTableRequest::OnExecute() {
}
status = ValidationUtil::ValidateTableIndexFileSize(index_file_size_);
fiu_do_on("CreateTableRequest.OnExecute.invalid_index_file_size",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -89,6 +92,10 @@ CreateTableRequest::OnExecute() {
// step 3: create table
status = DBWrapper::DB()->CreateTable(table_info);
fiu_do_on("CreateTableRequest.OnExecute.db_already_exist", status = Status(milvus::DB_ALREADY_EXIST, ""));
fiu_do_on("CreateTableRequest.OnExecute.create_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreateTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
// table could exist
if (status.code() == DB_ALREADY_EXIST) {

10
core/src/server/delivery/request/DeleteByDateRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
@ -54,8 +55,13 @@ DeleteByDateRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DeleteByDateRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("DeleteByDateRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DeleteByDateRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code(), DB_NOT_FOUND) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
} else {
return status;
@ -80,6 +86,8 @@ DeleteByDateRequest::OnExecute() {
ProfilerStart(fname.c_str());
#endif
status = DBWrapper::DB()->DropTable(table_name_, dates);
fiu_do_on("DeleteByDateRequest.OnExecute.drop_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

2
core/src/server/delivery/request/DescribeIndexRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -40,6 +41,7 @@ DescribeIndexRequest::Create(const std::shared_ptr<Context>& context, const std:
Status
DescribeIndexRequest::OnExecute() {
try {
fiu_do_on("DescribeIndexRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "DescribeIndexRequest(table=" + table_name_ + ")";
TimeRecorderAuto rc(hdr);

4
core/src/server/delivery/request/DescribeTableRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -53,6 +54,9 @@ DescribeTableRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DescribeTableRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DescribeTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}

4
core/src/server/delivery/request/DropIndexRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -38,6 +39,7 @@ DropIndexRequest::Create(const std::shared_ptr<Context>& context, const std::str
Status
DropIndexRequest::OnExecute() {
try {
fiu_do_on("DropIndexRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "DropIndexRequest(table=" + table_name_ + ")";
TimeRecorderAuto rc(hdr);
@ -49,6 +51,7 @@ DropIndexRequest::OnExecute() {
bool has_table = false;
status = DBWrapper::DB()->HasTable(table_name_, has_table);
fiu_do_on("DropIndexRequest.OnExecute.table_not_exist", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -59,6 +62,7 @@ DropIndexRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->DropIndex(table_name_);
fiu_do_on("DropIndexRequest.OnExecute.drop_index_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

9
core/src/server/delivery/request/DropPartitionRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
@ -60,6 +61,8 @@ DropPartitionRequest::OnExecute() {
if (!partition_name.empty()) {
status = ValidationUtil::ValidateTableName(partition_name);
fiu_do_on("DropPartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -68,6 +71,8 @@ DropPartitionRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = partition_name;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DropPartitionRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST,
@ -80,11 +85,15 @@ DropPartitionRequest::OnExecute() {
return DBWrapper::DB()->DropPartition(partition_name);
} else {
status = ValidationUtil::ValidateTableName(table_name);
fiu_do_on("DropPartitionRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
status = ValidationUtil::ValidatePartitionTags({partition_tag});
fiu_do_on("DropPartitionRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

6
core/src/server/delivery/request/DropTableRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <vector>
@ -52,6 +53,10 @@ DropTableRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("DropTableRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("DropTableRequest.OnExecute.describe_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("DropTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
@ -65,6 +70,7 @@ DropTableRequest::OnExecute() {
// step 3: Drop table
std::vector<DB_DATE> dates;
status = DBWrapper::DB()->DropTable(table_name_, dates);
fiu_do_on("DropTableRequest.OnExecute.drop_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

3
core/src/server/delivery/request/HasTableRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -50,6 +51,8 @@ HasTableRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->HasTable(table_name_, has_table_);
fiu_do_on("HasTableRequest.OnExecute.table_not_exist", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("HasTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}

12
core/src/server/delivery/request/InsertRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
@ -46,6 +47,7 @@ Status
InsertRequest::OnExecute() {
try {
int64_t vector_count = vectors_data_.vector_count_;
fiu_do_on("InsertRequest.OnExecute.throw_std_exception", throw std::exception());
std::string hdr = "InsertRequest(table=" + table_name_ + ", n=" + std::to_string(vector_count) +
", partition_tag=" + partition_tag_ + ")";
TimeRecorder rc(hdr);
@ -60,6 +62,7 @@ InsertRequest::OnExecute() {
"The vector array is empty. Make sure you have entered vector records.");
}
fiu_do_on("InsertRequest.OnExecute.id_array_error", vectors_data_.id_array_.resize(vector_count + 1));
if (!vectors_data_.id_array_.empty()) {
if (vectors_data_.id_array_.size() != vector_count) {
return Status(SERVER_ILLEGAL_VECTOR_ID,
@ -71,6 +74,8 @@ InsertRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("InsertRequest.OnExecute.db_not_found", status = Status(milvus::DB_NOT_FOUND, ""));
fiu_do_on("InsertRequest.OnExecute.describe_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
@ -82,12 +87,16 @@ InsertRequest::OnExecute() {
// step 3: check table flag
// all user provide id, or all internal id
bool user_provide_ids = !vectors_data_.id_array_.empty();
fiu_do_on("InsertRequest.OnExecute.illegal_vector_id", user_provide_ids = false;
table_info.flag_ = engine::meta::FLAG_MASK_HAS_USERID);
// user already provided id before, all insert action require user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) {
return Status(SERVER_ILLEGAL_VECTOR_ID,
"Table vector IDs are user-defined. Please provide IDs for all vectors of this table.");
}
fiu_do_on("InsertRequest.OnExecute.illegal_vector_id2", user_provide_ids = true;
table_info.flag_ = engine::meta::FLAG_MASK_NO_USERID);
// user didn't provided id before, no need to provide user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) {
return Status(
@ -114,6 +123,7 @@ InsertRequest::OnExecute() {
"The vector dimension must be equal to the table dimension.");
}
fiu_do_on("InsertRequest.OnExecute.invalid_dim", table_info.dimension_ = -1);
if (vectors_data_.float_data_.size() / vector_count != table_info.dimension_) {
return Status(SERVER_INVALID_VECTOR_DIMENSION,
"The vector dimension must be equal to the table dimension.");
@ -140,11 +150,13 @@ InsertRequest::OnExecute() {
rc.RecordSection("prepare vectors data");
status = DBWrapper::DB()->InsertVectors(table_name_, partition_tag_, vectors_data_);
fiu_do_on("InsertRequest.OnExecute.insert_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
auto ids_size = vectors_data_.id_array_.size();
fiu_do_on("InsertRequest.OnExecute.invalid_ids_size", ids_size = vec_count - 1);
if (ids_size != vec_count) {
std::string msg =
"Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id";

4
core/src/server/delivery/request/PreloadTableRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -49,6 +50,9 @@ PreloadTableRequest::OnExecute() {
// step 2: check table existence
status = DBWrapper::DB()->PreloadTable(table_name_);
fiu_do_on("PreloadTableRequest.OnExecute.preload_table_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("PreloadTableRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
return status;
}

12
core/src/server/delivery/request/SearchRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
namespace milvus {
@ -53,6 +54,7 @@ SearchRequest::Create(const std::shared_ptr<Context>& context, const std::string
Status
SearchRequest::OnExecute() {
try {
fiu_do_on("SearchRequest.OnExecute.throw_std_exception", throw std::exception());
uint64_t vector_count = vectors_data_.vector_count_;
auto pre_query_ctx = context_->Child("Pre query");
@ -71,6 +73,7 @@ SearchRequest::OnExecute() {
engine::meta::TableSchema table_info;
table_info.table_id_ = table_name_;
status = DBWrapper::DB()->DescribeTable(table_info);
fiu_do_on("SearchRequest.OnExecute.describe_table_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
@ -117,11 +120,13 @@ SearchRequest::OnExecute() {
}
} else {
// check prepared float data
fiu_do_on("SearchRequest.OnExecute.invalod_rowrecord_array",
vector_count = vectors_data_.float_data_.size() + 1);
if (vectors_data_.float_data_.size() % vector_count != 0) {
return Status(SERVER_INVALID_ROWRECORD_ARRAY,
"The vector dimension must be equal to the table dimension.");
}
fiu_do_on("SearchRequest.OnExecute.invalid_dim", table_info.dimension_ = -1);
if (vectors_data_.float_data_.size() / vector_count != table_info.dimension_) {
return Status(SERVER_INVALID_VECTOR_DIMENSION,
"The vector dimension must be equal to the table dimension.");
@ -144,6 +149,8 @@ SearchRequest::OnExecute() {
if (file_id_list_.empty()) {
status = ValidationUtil::ValidatePartitionTags(partition_list_);
fiu_do_on("SearchRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -160,10 +167,11 @@ SearchRequest::OnExecute() {
#endif
rc.RecordSection("search vectors from engine");
fiu_do_on("SearchRequest.OnExecute.query_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
fiu_do_on("SearchRequest.OnExecute.empty_result_ids", result_ids.clear());
if (result_ids.empty()) {
return Status::OK(); // empty table
}

5
core/src/server/delivery/request/ShowPartitionsRequest.cpp
View File

@ -21,6 +21,7 @@
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include <fiu-local.h>
#include <memory>
#include <vector>
@ -44,6 +45,8 @@ ShowPartitionsRequest::OnExecute() {
TimeRecorderAuto rc(hdr);
auto status = ValidationUtil::ValidateTableName(table_name_);
fiu_do_on("ShowPartitionsRequest.OnExecute.invalid_table_name",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
@ -60,6 +63,8 @@ ShowPartitionsRequest::OnExecute() {
std::vector<engine::meta::TableSchema> schema_array;
status = DBWrapper::DB()->ShowPartitions(table_name_, schema_array);
fiu_do_on("ShowPartitionsRequest.OnExecute.show_partition_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}

8
core/src/server/delivery/request/ShowTablesRequest.cpp
View File

@ -20,6 +20,7 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <fiu-local.h>
#include <memory>
#include <string>
#include <vector>
@ -41,9 +42,10 @@ ShowTablesRequest::OnExecute() {
TimeRecorderAuto rc("ShowTablesRequest");
std::vector<engine::meta::TableSchema> schema_array;
auto statuts = DBWrapper::DB()->AllTables(schema_array);
if (!statuts.ok()) {
return statuts;
auto status = DBWrapper::DB()->AllTables(schema_array);
fiu_do_on("ShowTablesRequest.OnExecute.show_tables_fail", status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
for (auto& schema : schema_array) {

3
core/src/server/grpc_impl/GrpcRequestHandler.cpp
View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <memory>
#include <unordered_map>
#include <vector>
@ -290,6 +290,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc:
// step 3: search vectors
std::vector<std::string> file_ids;
TopKQueryResult result;
fiu_do_on("GrpcRequestHandler.Search.not_empty_file_ids", file_ids.emplace_back("test_file_id"));
Status status = request_handler_.Search(context_map_[context], request->table_name(), vectors, ranges,
request->topk(), request->nprobe(), partitions, file_ids, result);

8
core/src/utils/CommonUtil.cpp
View File

@ -42,6 +42,8 @@
#define THREAD_MULTIPLY_CPU 1
#endif
#include <fiu-local.h>
namespace milvus {
namespace server {
@ -62,6 +64,8 @@ CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("CommonUtil.GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
@ -95,6 +99,7 @@ CommonUtil::CreateDirectory(const std::string& path) {
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
Status err_status = CreateDirectory(parent_path.string());
fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!err_status.ok()) {
return err_status;
}
@ -105,6 +110,7 @@ CommonUtil::CreateDirectory(const std::string& path) {
}
int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
if (makeOK != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
}
@ -183,6 +189,7 @@ CommonUtil::GetExePath() {
const size_t buf_len = 1024;
char buf[buf_len];
size_t cnt = readlink("/proc/self/exe", buf, buf_len);
fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
if (cnt < 0 || cnt >= buf_len) {
return "";
}
@ -190,6 +197,7 @@ CommonUtil::GetExePath() {
buf[cnt] = '\0';
std::string exe_path = buf;
fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
if (exe_path.rfind('/') != exe_path.length() - 1) {
std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
return sub_str + "/";

7
core/src/utils/StringHelpFunctions.cpp
View File

@ -17,6 +17,7 @@
#include "utils/StringHelpFunctions.h"
#include <fiu-local.h>
#include <regex>
#include <string>
@ -104,6 +105,8 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
last = index + 1;
std::string postfix = process_str.substr(last);
index = postfix.find_first_of(quote, 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
if (index == std::string::npos) {
return Status(SERVER_UNEXPECTED_ERROR, "");
}
@ -112,6 +115,9 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
last = index + 1;
index = postfix.find_first_of(delimeter, last);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
if (index != std::string::npos) {
if (index > last) {
append_prefix += postfix.substr(last, index - last);
@ -120,6 +126,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
append_prefix += postfix.substr(last);
}
result.emplace_back(append_prefix);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
if (last == postfix.length()) {
return Status::OK();

3
core/src/utils/ThreadPool.h
View File

@ -17,6 +17,7 @@
#pragma once
#include <fiu-local.h>
#include <condition_variable>
#include <functional>
#include <future>
@ -89,7 +90,7 @@ ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_o
auto task = std::make_shared<std::packaged_task<return_type()> >(
std::bind(std::forward<F>(f), std::forward<Args>(args)...));
fiu_do_on("ThreadPool.enqueue.stop_is_true", stop = true);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex_);

3
core/src/utils/TimeRecorder.cpp
View File

@ -24,8 +24,7 @@ TimeRecorder::TimeRecorder(const std::string& header, int64_t log_level) : heade
start_ = last_ = stdclock::now();
}
TimeRecorder::~TimeRecorder() {
}
TimeRecorder::~TimeRecorder() = default;
std::string
TimeRecorder::GetTimeSpanStr(double span) {

2
core/src/utils/TimeRecorder.h
View File

@ -54,7 +54,7 @@ class TimeRecorderAuto : public TimeRecorder {
public:
explicit TimeRecorderAuto(const std::string& header, int64_t log_level = 1);
~TimeRecorderAuto();
~TimeRecorderAuto() override;
};
} // namespace milvus

8
core/src/utils/ValidationUtil.cpp
View File

@ -28,6 +28,7 @@
#endif
#include <fiu-local.h>
#include <algorithm>
#include <cmath>
#include <regex>
@ -245,6 +246,8 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("ValidationUtil.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
SERVER_LOG_ERROR << msg;
@ -265,6 +268,8 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
Status
ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) {
fiu_return_on("ValidationUtil.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
@ -285,6 +290,7 @@ ValidationUtil::ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("ValidationUtil.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
@ -309,6 +315,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string& str) {
}
try {
int64_t value = std::stol(str);
fiu_do_on("ValidationUtil.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
@ -320,6 +327,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string& str) {
Status
ValidationUtil::ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||

3
core/src/wrapper/ConfAdapter.cpp
View File

@ -17,6 +17,7 @@
#include "wrapper/ConfAdapter.h"
#include <fiu-local.h>
#include <cmath>
#include <memory>
#include <vector>
@ -160,7 +161,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) {
}
}
}
fiu_do_on("IVFPQConfAdapter.Match.empty_resset", resset.clear());
if (resset.empty()) {
// todo(linxj): throw exception here.
WRAPPER_LOG_ERROR << "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-"

3
core/src/wrapper/KnowhereResource.cpp
View File

@ -23,6 +23,7 @@
#include "scheduler/Utils.h"
#include "server/Config.h"
#include <fiu-local.h>
#include <map>
#include <set>
#include <string>
@ -43,7 +44,7 @@ KnowhereResource::Initialize() {
s = config.GetGpuResourceConfigEnable(enable_gpu);
if (!s.ok())
return s;
fiu_do_on("KnowhereResource.Initialize.disable_gpu", enable_gpu = false);
if (not enable_gpu)
return Status::OK();

17
core/src/wrapper/VecImpl.cpp
View File

@ -32,6 +32,7 @@
#endif
#include <fiu-local.h>
/*
* no parameter check in this layer.
* only responsible for index combination
@ -46,6 +47,8 @@ VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, c
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("VecIndexImpl.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.BuildAll.throw_std_exception", throw std::exception());
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
@ -66,7 +69,8 @@ Status
VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("VecIndexImpl.Add.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.Add.throw_std_exception", throw std::exception());
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
@ -86,6 +90,9 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
Config search_cfg = cfg;
fiu_do_on("VecIndexImpl.Search.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndexImpl.Search.throw_std_exception", throw std::exception());
auto res = index_->Search(dataset, search_cfg);
//{
// auto& ids = ids_array;
@ -154,7 +161,7 @@ VecIndexImpl::GetType() const {
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
// TODO(linxj): exception handle
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
@ -168,7 +175,7 @@ VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config& cfg) {
// TODO(linxj): exception handle
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
@ -217,6 +224,8 @@ BFIndex::GetRawIds() {
ErrorCode
BFIndex::Build(const Config& cfg) {
try {
fiu_do_on("BFIndex.Build.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("BFIndex.Build.throw_std_exception", throw std::exception());
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException& e) {
@ -235,6 +244,8 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
fiu_do_on("BFIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("BFIndex.BuildAll.throw_std_exception", throw std::exception());
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);

10
core/src/wrapper/VecIndex.cpp
View File

@ -49,6 +49,8 @@
#include "wrapper/gpu/GPUVecImpl.h"
#endif
#include <fiu-local.h>
namespace milvus {
namespace engine {
@ -142,6 +144,7 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
config.GetGpuResourceConfigEnable(gpu_resource_enable);
if (gpu_resource_enable) {
index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
fiu_do_on("GetVecIndexFactory.IVFSQHybrid.mock", index = std::make_shared<knowhere::IVF>());
return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
} else {
throw Exception(DB_ERROR, "No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID");
@ -171,6 +174,8 @@ LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binar
VecIndexPtr
read_index(const std::string& location) {
fiu_return_on("read_null_index", nullptr);
fiu_do_on("vecIndex.throw_read_exception", throw std::exception());
TimeRecorder recorder("read_index");
knowhere::BinarySet load_data_list;
@ -242,6 +247,11 @@ write_index(VecIndexPtr index, const std::string& location) {
auto binaryset = index->Serialize();
auto index_type = index->GetType();
fiu_do_on("VecIndex.write_index.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("VecIndex.write_index.throw_std_exception", throw std::exception());
fiu_do_on("VecIndex.write_index.throw_no_space_exception",
throw Exception(SERVER_INVALID_ARGUMENT, "No space left on device"));
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);

19
core/src/wrapper/gpu/GPUVecImpl.cpp
View File

@ -26,6 +26,7 @@
#include "utils/Log.h"
#include "wrapper/VecImpl.h"
#include <fiu-local.h>
/*
* no parameter check in this layer.
* only responible for index combination
@ -39,9 +40,11 @@ Status
IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
fiu_do_on("IVFMixIndex.BuildAll.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFMixIndex.BuildAll.throw_std_exception", throw std::exception());
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
@ -81,12 +84,16 @@ IVFHybridIndex::LoadQuantizer(const Config& conf) {
return new_idx->LoadQuantizer(conf);
} else {
WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type);
return nullptr;
}
}
Status
IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) {
try {
fiu_do_on("IVFHybridIndex.SetQuantizer.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.SetQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
new_idx->SetQuantizer(q);
@ -107,6 +114,9 @@ IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) {
Status
IVFHybridIndex::UnsetQuantizer() {
try {
fiu_do_on("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.UnsetQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
new_idx->UnsetQuantizer();
@ -127,6 +137,9 @@ IVFHybridIndex::UnsetQuantizer() {
VecIndexPtr
IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
try {
fiu_do_on("IVFHybridIndex.LoadData.throw_knowhere_exception", throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.LoadData.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
return std::make_shared<IVFHybridIndex>(new_idx->LoadData(q, conf), type);
@ -144,6 +157,10 @@ IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
std::pair<VecIndexPtr, knowhere::QuantizerPtr>
IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) {
try {
fiu_do_on("IVFHybridIndex.CopyToGpuWithQuantizer.throw_knowhere_exception",
throw knowhere::KnowhereException(""));
fiu_do_on("IVFHybridIndex.CopyToGpuWithQuantizer.throw_std_exception", throw std::exception());
// TODO(linxj): Hardcode here
if (auto hybrid_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg);

274
core/unittest/db/test_db.cpp
View File

@ -16,6 +16,8 @@
// under the License.
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <random>
@ -40,7 +42,6 @@ static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
milvus::engine::meta::TableSchema
BuildTableSchema() {
milvus::engine::meta::TableSchema table_info;
@ -125,6 +126,18 @@ TEST_F(DBTest, CONFIG_TEST) {
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 0);
}
{
fiu_init(0);
fiu_enable("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange", 1, NULL, 0);
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf("swap", "disk:"));
fiu_disable("ArchiveConf.ParseCritirias.OptionsParseCritiriasOutOfRange");
}
{
fiu_enable("ArchiveConf.ParseCritirias.empty_tokens", 1, NULL, 0);
milvus::engine::ArchiveConf conf("swap", "");
ASSERT_TRUE(conf.GetCriterias().empty());
fiu_disable("ArchiveConf.ParseCritirias.empty_tokens");
}
{
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "disk:a"));
@ -228,6 +241,19 @@ TEST_F(DBTest, DB_TEST) {
stat = db_->GetTableRowCount(TABLE_NAME, count);
ASSERT_TRUE(stat.ok());
ASSERT_GT(count, 0);
// test invalid build db
{
auto options = GetOptions();
options.meta_.backend_uri_ = "dummy";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
options.meta_.backend_uri_ = "mysql://root:123456@127.0.0.1:3306/test";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
options.meta_.backend_uri_ = "dummy://root:123456@127.0.0.1:3306/test";
ASSERT_ANY_THROW(milvus::engine::DBFactory::Build(options));
}
}
TEST_F(DBTest, SEARCH_TEST) {
@ -341,6 +367,18 @@ TEST_F(DBTest, SEARCH_TEST) {
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
FIU_ENABLE_FIU("DBImpl.QueryByFileID.empty_files_array");
stat = db_->QueryByFileID(dummy_context_, TABLE_NAME, file_ids, k, 10, xq, dates, result_ids,
result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.QueryByFileID.empty_files_array");
}
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
@ -352,6 +390,13 @@ TEST_F(DBTest, SEARCH_TEST) {
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_TRUE(stat.ok());
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, 10, xq, result_ids, result_distances);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
}
#ifdef CUSTOMIZATION
@ -389,6 +434,8 @@ TEST_F(DBTest, SEARCH_TEST) {
}
TEST_F(DBTest, PRELOADTABLE_TEST) {
fiu_init(0);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
@ -418,6 +465,32 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
ASSERT_TRUE(stat.ok());
int64_t cur_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
ASSERT_TRUE(prev_cache_usage < cur_cache_usage);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
//create a partition
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_TRUE(stat.ok());
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("DBImpl.PreloadTable.null_engine");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.null_engine");
FIU_ENABLE_FIU("DBImpl.PreloadTable.exceed_cache");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.exceed_cache");
FIU_ENABLE_FIU("DBImpl.PreloadTable.engine_throw_exception");
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.PreloadTable.engine_throw_exception");
}
TEST_F(DBTest, SHUTDOWN_TEST) {
@ -430,6 +503,26 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->DescribeTable(table_info);
ASSERT_FALSE(stat.ok());
stat = db_->UpdateTableFlag(TABLE_NAME, 0);
ASSERT_FALSE(stat.ok());
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_FALSE(stat.ok());
stat = db_->DropPartition("part0");
ASSERT_FALSE(stat.ok());
stat = db_->DropPartitionByTag(TABLE_NAME, "0");
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(TABLE_NAME, partition_schema_array);
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> table_infos;
stat = db_->AllTables(table_infos);
ASSERT_EQ(stat.code(), milvus::DB_ERROR);
bool has_table = false;
stat = db_->HasTable(table_info.table_id_, has_table);
ASSERT_FALSE(stat.ok());
@ -452,6 +545,9 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->DescribeIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
stat = db_->DropIndex(TABLE_NAME);
ASSERT_FALSE(stat.ok());
std::vector<std::string> tags;
milvus::engine::meta::DatesT dates;
milvus::engine::ResultIds result_ids;
@ -464,10 +560,127 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
result_distances);
ASSERT_FALSE(stat.ok());
stat = db_->Query(dummy_context_, table_info.table_id_, tags, 1, 1,
milvus::engine::VectorsData(), result_ids, result_distances);
ASSERT_FALSE(stat.ok());
stat = db_->DropTable(table_info.table_id_, dates);
ASSERT_FALSE(stat.ok());
}
TEST_F(DBTest, BACK_TIMER_THREAD_1) {
fiu_init(0);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::Status stat;
//test background timer thread
{
FIU_ENABLE_FIU("DBImpl.StartMetricTask.InvalidTotalCache");
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToMerge.throw_exception");
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.StartMetricTask.InvalidTotalCache");
fiu_disable("SqliteMetaImpl.FilesToMerge.throw_exception");
}
FIU_ENABLE_FIU("DBImpl.StartMetricTask.InvalidTotalCache");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.StartMetricTask.InvalidTotalCache");
}
TEST_F(DBTest, BACK_TIMER_THREAD_2) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
}
TEST_F(DBTest, BACK_TIMER_THREAD_3) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("DBImpl.MergeFiles.Serialize_ThrowException");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.MergeFiles.Serialize_ThrowException");
}
TEST_F(DBTest, BACK_TIMER_THREAD_4) {
fiu_init(0);
milvus::Status stat;
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
stat = db_->CreateTable(table_info);
ASSERT_TRUE(stat.ok());
//insert some vector to create some tablefiles
int64_t nb = VECTOR_COUNT;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
int loop = 10;
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, "", xb);
ASSERT_EQ(xb.id_array_.size(), nb);
}
FIU_ENABLE_FIU("DBImpl.MergeFiles.Serialize_ErrorStatus");
db_->Start();
std::this_thread::sleep_for(std::chrono::seconds(2));
db_->Stop();
fiu_disable("DBImpl.MergeFiles.Serialize_ErrorStatus");
}
TEST_F(DBTest, INDEX_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
@ -489,6 +702,18 @@ TEST_F(DBTest, INDEX_TEST) {
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTableIndex.throw_exception");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.DescribeTableIndex.throw_exception");
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
FIU_ENABLE_FIU("DBImpl.UpdateTableIndexRecursively.fail_update_table_index");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.UpdateTableIndexRecursively.fail_update_table_index");
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
@ -542,6 +767,10 @@ TEST_F(DBTest, PARTITION_TEST) {
db_->InsertVectors(table_name, partition_tag, xb);
ASSERT_EQ(vector_ids.size(), INSERT_BATCH);
//insert data into not existed partition
stat = db_->InsertVectors(TABLE_NAME, "notexist", xb);
ASSERT_FALSE(stat.ok());
}
// duplicated partition is not allowed
@ -563,10 +792,31 @@ TEST_F(DBTest, PARTITION_TEST) {
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.BuildTableIndexRecursively.fail_build_table_Index_for_partition");
FIU_ENABLE_FIU("DBImpl.BuildTableIndexRecursively.not_empty_err_msg");
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.BuildTableIndexRecursively.not_empty_err_msg");
uint64_t row_count = 0;
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, INSERT_BATCH * PARTITION_COUNT);
FIU_ENABLE_FIU("SqliteMetaImpl.Count.throw_exception");
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_FALSE(stat.ok());
fiu_disable("SqliteMetaImpl.Count.throw_exception");
FIU_ENABLE_FIU("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition");
stat = db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.GetTableRowCountRecursively.fail_get_table_rowcount_for_partition");
}
{ // search
@ -607,6 +857,16 @@ TEST_F(DBTest, PARTITION_TEST) {
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
FIU_ENABLE_FIU("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_info.table_id_);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
FIU_ENABLE_FIU("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_info.table_id_);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableIndexRecursively.fail_drop_table_Index_for_partition");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
@ -684,6 +944,18 @@ TEST_F(DBTest2, DELETE_TEST) {
stat = db_->CreateIndex(TABLE_NAME, index);
std::vector<milvus::engine::meta::DateT> dates;
//create partition, drop table will drop partition recursively
stat = db_->CreatePartition(TABLE_NAME, "part0", "0");
ASSERT_TRUE(stat.ok());
//fail drop table
fiu_init(0);
FIU_ENABLE_FIU("DBImpl.DropTableRecursively.failed");
stat = db_->DropTable(TABLE_NAME, dates);
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.DropTableRecursively.failed");
stat = db_->DropTable(TABLE_NAME, dates);
std::this_thread::sleep_for(std::chrono::seconds(2));
ASSERT_TRUE(stat.ok());

115
core/unittest/db/test_db_mysql.cpp
View File

@ -16,7 +16,8 @@
// under the License.
#include <gtest/gtest.h>
#include <fiu-control.h>
#include <fiu-local.h>
#include <boost/filesystem.hpp>
#include <random>
#include <thread>
@ -204,6 +205,17 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
}
ASSERT_TRUE(bfound);
fiu_init(0);
FIU_ENABLE_FIU("MySQLMetaImpl.AllTable.null_connection");
stat = db_->AllTables(table_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.AllTable.throw_exception");
stat = db_->AllTables(table_schema_array);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.AllTable.null_connection");
fiu_disable("MySQLMetaImpl.AllTable.throw_exception");
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
@ -231,6 +243,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * milvus::engine::G);
FIU_ENABLE_FIU("MySQLMetaImpl.Size.null_connection");
stat = db_->Size(size);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.Size.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.Size.throw_exception");
stat = db_->Size(size);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.Size.throw_exception");
}
TEST_F(MySqlDBTest, DELETE_TEST) {
@ -288,6 +309,12 @@ TEST_F(MySqlDBTest, PARTITION_TEST) {
stat = db_->CreatePartition(table_name, partition_name, partition_tag);
ASSERT_TRUE(stat.ok());
fiu_init(0);
FIU_ENABLE_FIU("MySQLMetaImpl.CreatePartition.aleady_exist");
stat = db_->CreatePartition(table_name, partition_name, partition_tag);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.CreatePartition.aleady_exist");
// not allow nested partition
stat = db_->CreatePartition(partition_name, "dumy", "dummy");
ASSERT_FALSE(stat.ok());
@ -366,16 +393,84 @@ TEST_F(MySqlDBTest, PARTITION_TEST) {
ASSERT_EQ(result_ids.size() / topk, nq);
}
stat = db_->DropPartition(table_name + "_0");
ASSERT_TRUE(stat.ok());
fiu_init(0);
{
//create partition with dummy name
stat = db_->CreatePartition(table_name, "", "6");
ASSERT_TRUE(stat.ok());
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
// ensure DescribeTable failed
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
stat = db_->CreatePartition(table_name, "", "7");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
//Drop partition will failed,since it firstly drop partition meta table.
FIU_ENABLE_FIU("MySQLMetaImpl.DropTable.null_connection");
stat = db_->DropPartition(table_name + "_5");
//TODO(sjh): add assert expr, since DropPartion always return Status::OK() for now.
//ASSERT_TRUE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTable.null_connection");
milvus::engine::meta::DatesT dates;
stat = db_->DropTable(table_name, dates);
ASSERT_TRUE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT + 1);
FIU_ENABLE_FIU("MySQLMetaImpl.ShowPartitions.null_connection");
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.ShowPartitions.throw_exception");
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_FALSE(stat.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.DropTable.throw_exception");
stat = db_->DropPartition(table_name + "_4");
fiu_disable("MySQLMetaImpl.DropTable.throw_exception");
stat = db_->DropPartition(table_name + "_0");
ASSERT_TRUE(stat.ok());
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.GetPartitionName.null_connection");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.GetPartitionName.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.GetPartitionName.throw_exception");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.GetPartitionName.throw_exception");
stat = db_->DropPartitionByTag(table_name, "1");
ASSERT_TRUE(stat.ok());
stat = db_->CreatePartition(table_name, table_name + "_1", "1");
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.null_connection");
stat = db_->DropPartition(table_name + "_1");
fiu_disable("MySQLMetaImpl.DeleteTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.throw_exception");
stat = db_->DropPartition(table_name + "_1");
fiu_disable("MySQLMetaImpl.DeleteTableFiles.throw_exception");
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.DropTableIndex.null_connection");
stat = db_->DropIndex(table_name);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTableIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DropTableIndex.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DropTableIndex.throw_exception");
stat = db_->DropIndex(table_name);
ASSERT_TRUE(stat.ok());
}
}

152
core/unittest/db/test_engine.cpp
View File

@ -22,6 +22,8 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "db/utils.h"
#include <fiu-local.h>
#include <fiu-control.h>
TEST_F(EngineTest, FACTORY_TEST) {
{
@ -62,29 +64,49 @@ TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
fiu_init(0);
// test ExecutionEngineImpl constructor when create vecindex failed
FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.invalid_type");
ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024));
fiu_disable("ExecutionEngineImpl.CreatetVecIndex.invalid_type");
}
{
// test ExecutionEngineImpl constructor when build BFindex failed
FIU_ENABLE_FIU("BFIndex.Build.throw_knowhere_exception");
ASSERT_ANY_THROW(milvus::engine::EngineFactory::Build(
512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
milvus::engine::MetricType::L2, 1024));
fiu_disable("BFIndex.Build.throw_knowhere_exception");
}
}
TEST_F(EngineTest, ENGINE_IMPL_TEST) {
fiu_init(0);
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
@ -108,18 +130,122 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Dimension(), dimension);
ASSERT_EQ(engine_ptr->Count(), ids.size());
ASSERT_EQ(engine_ptr->GetLocation(), file_path);
ASSERT_EQ(engine_ptr->IndexMetricType(), milvus::engine::MetricType::IP);
ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::INVALID));
FIU_ENABLE_FIU("VecIndexImpl.BuildAll.throw_knowhere_exception");
ASSERT_ANY_THROW(engine_ptr->BuildIndex(file_path, milvus::engine::EngineType::SPTAG_KDT));
fiu_disable("VecIndexImpl.BuildAll.throw_knowhere_exception");
auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
#ifndef MILVUS_GPU_VERSION
//PQ don't support IP In gpu version
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
#endif
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
engine_ptr->BuildIndex("/tmp/milvus_index_SPTAG_BKT", milvus::engine::EngineType::SPTAG_BKT);
#ifdef MILVUS_GPU_VERSION
FIU_ENABLE_FIU("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled");
engine_ptr->BuildIndex("/tmp/milvus_index_NSG_MIX", milvus::engine::EngineType::NSG_MIX);
engine_ptr->BuildIndex("/tmp/milvus_index_6", milvus::engine::EngineType::FAISS_IVFFLAT);
engine_ptr->BuildIndex("/tmp/milvus_index_7", milvus::engine::EngineType::FAISS_IVFSQ8);
ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_8", milvus::engine::EngineType::FAISS_IVFSQ8H));
ASSERT_ANY_THROW(engine_ptr->BuildIndex("/tmp/milvus_index_9", milvus::engine::EngineType::FAISS_PQ));
fiu_disable("ExecutionEngineImpl.CreatetVecIndex.gpu_res_disabled");
#endif
//merge self
status = engine_ptr->Merge(file_path);
ASSERT_FALSE(status.ok());
// FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
// status = engine_ptr->Merge("/tmp/milvus_index_2");
// ASSERT_FALSE(status.ok());
// fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
FIU_ENABLE_FIU("vecIndex.throw_read_exception");
status = engine_ptr->Merge("dummy");
ASSERT_FALSE(status.ok());
fiu_disable("vecIndex.throw_read_exception");
//CPU version invoke CopyToCpu will fail
status = engine_ptr->CopyToCpu();
ASSERT_FALSE(status.ok());
#ifdef MILVUS_GPU_VERSION
status = engine_ptr->CopyToGpu(0, false);
// ASSERT_TRUE(status.ok());
ASSERT_TRUE(status.ok());
status = engine_ptr->GpuCache(0);
ASSERT_TRUE(status.ok());
status = engine_ptr->CopyToGpu(0, false);
ASSERT_TRUE(status.ok());
// auto new_engine = engine_ptr->Clone();
// ASSERT_EQ(new_engine->Dimension(), dimension);
// ASSERT_EQ(new_engine->Count(), ids.size());
status = engine_ptr->CopyToCpu();
// ASSERT_TRUE(status.ok());
auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
// ASSERT_TRUE(status.ok());
status = engine_ptr->CopyToCpu();
ASSERT_TRUE(status.ok());
engine_ptr->CopyToCpu();
ASSERT_TRUE(status.ok());
#endif
}
TEST_F(EngineTest, ENGINE_IMPL_NULL_INDEX_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
fiu_init(0); // init
fiu_enable("read_null_index", 1, NULL, 0);
engine_ptr->Load(true);
auto count = engine_ptr->Count();
ASSERT_EQ(count, 0);
auto dim = engine_ptr->Dimension();
ASSERT_EQ(dim, dimension);
auto status = engine_ptr->Merge("/tmp/milvus_index_2");
ASSERT_FALSE(status.ok());
auto build_index = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IDMAP);
ASSERT_EQ(build_index, nullptr);
int64_t n = 0;
const float* data = nullptr;
int64_t k = 10;
int64_t nprobe = 0;
float* distances = nullptr;
int64_t* labels = nullptr;
bool hybrid = false;
status = engine_ptr->Search(n, data, k, nprobe, distances, labels, hybrid);
ASSERT_FALSE(status.ok());
fiu_disable("read_null_index");
}
TEST_F(EngineTest, ENGINE_IMPL_THROW_EXCEPTION_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/invalid_file";
fiu_init(0); // init
fiu_enable("ValidateStringNotBool", 1, NULL, 0);
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
fiu_disable("ValidateStringNotBool");
fiu_init(0); // init
fiu_enable("vecIndex.throw_read_exception", 1, NULL, 0);
engine_ptr->Load(true);
engine_ptr->CopyToGpu(0, true);
engine_ptr->CopyToCpu();
fiu_disable("vecIndex.throw_read_exception");
}

66
core/unittest/db/test_mem.cpp
View File

@ -22,6 +22,9 @@
#include <iostream>
#include <random>
#include <thread>
#include <fiu-control.h>
#include <fiu-local.h>
#include <src/db/DBFactory.h>
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
@ -98,10 +101,23 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(num_vectors_added, 50);
ASSERT_EQ(source.GetVectorIds().size(), 100);
auto current_num_vectors_added = source.GetNumVectorsAdded();
ASSERT_EQ(current_num_vectors_added, 100);
vectors.id_array_ = source.GetVectorIds();
ASSERT_EQ(vectors.id_array_.size(), 100);
fiu_init(0);
FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
status = source.Add(execution_engine_, table_file_schema, 60, num_vectors_added);
ASSERT_FALSE(status.ok());
fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
}
TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
auto options = GetOptions();
fiu_init(0);
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
@ -115,7 +131,6 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
milvus::engine::VectorSourcePtr source = std::make_shared<milvus::engine::VectorSource>(vectors_100);
milvus::engine::IDNumbers vector_ids;
status = mem_table_file.Add(source);
ASSERT_TRUE(status.ok());
@ -130,12 +145,38 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(vectors_128M);
vector_ids.clear();
status = mem_table_file.Add(source_128M);
ASSERT_EQ(source_128M->GetVectorIds().size(), n_max - n_100);
ASSERT_TRUE(mem_table_file.IsFull());
//mem_table_file has no memory left = 0
status = mem_table_file.Add(source_128M);
ASSERT_TRUE(status.ok());
{
//test fail create table file
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
milvus::engine::MemTableFile mem_table_file_1(GetTableName(), impl_, options);
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
status = mem_table_file_1.Add(source);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_ERROR);
}
{
options.insert_cache_immediately_ = true;
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
table_schema.table_id_ = "faiss_pq";
table_schema.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
milvus::engine::MemTableFile mem_table_file_1("faiss_pq", impl_, options);
mem_table_file_1.Serialize();
}
}
TEST_F(MemManagerTest, MEM_TABLE_TEST) {
@ -154,7 +195,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::MemTable mem_table(GetTableName(), impl_, options);
milvus::engine::IDNumbers vector_ids;
status = mem_table.Add(source_100);
ASSERT_TRUE(status.ok());
ASSERT_EQ(source_100->GetVectorIds().size(), 100);
@ -168,7 +208,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::VectorsData vectors_128M;
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(vectors_128M);
status = mem_table.Add(source_128M);
@ -187,7 +226,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
milvus::engine::VectorSourcePtr source_1G = std::make_shared<milvus::engine::VectorSource>(vectors_1G);
vector_ids.clear();
status = mem_table.Add(source_1G);
ASSERT_TRUE(status.ok());
@ -198,6 +236,24 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
status = mem_table.Serialize();
ASSERT_TRUE(status.ok());
milvus::engine::VectorsData vectors_10;
BuildVectors(10, vectors_10);
milvus::engine::VectorSourcePtr source_10 = std::make_shared<milvus::engine::VectorSource>(vectors_10);
fiu_init(0);
FIU_ENABLE_FIU("VecIndexImpl.Add.throw_knowhere_exception");
status = mem_table.Add(source_10);
ASSERT_FALSE(status.ok());
fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
status = mem_table.Add(source_10);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFile.throw_exception");
status = mem_table.Serialize();
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.UpdateTableFile.throw_exception");
}
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {

383
core/unittest/db/test_meta.cpp
View File

@ -25,6 +25,10 @@
#include <stdlib.h>
#include <time.h>
#include <thread>
#include <fiu-local.h>
#include <fiu-control.h>
#include <boost/filesystem/operations.hpp>
#include "src/db/OngoingFileChecker.h"
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
@ -49,11 +53,352 @@ TEST_F(MetaTest, TABLE_TEST) {
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
status = impl_->DropTable(table.table_id_);
ASSERT_TRUE(status.ok());
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), milvus::DB_ERROR);
table.table_id_ = "";
status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
}
TEST_F(MetaTest, FALID_TEST) {
fiu_init(0);
auto options = GetOptions();
auto table_id = "meta_test_table";
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
milvus::Status status;
{
FIU_ENABLE_FIU("SqliteMetaImpl.ValidateMetaSchema.NullConnection");
milvus::engine::meta::SqliteMetaImpl impl(options.meta_);
fiu_disable("SqliteMetaImpl.ValidateMetaSchema.NullConnection");
}
{
//failed initialize
auto options_1 = options;
options_1.meta_.path_ = options.meta_.path_ + "1";
if (boost::filesystem::is_directory(options_1.meta_.path_)) {
boost::filesystem::remove_all(options_1.meta_.path_);
}
FIU_ENABLE_FIU("SqliteMetaImpl.Initialize.fail_create_directory");
ASSERT_ANY_THROW(milvus::engine::meta::SqliteMetaImpl impl(options_1.meta_));
fiu_disable("SqliteMetaImpl.Initialize.fail_create_directory");
boost::filesystem::remove_all(options_1.meta_.path_);
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTable.throw_exception");
status = impl_->CreateTable(table);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CreateTable.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTable.insert_throw_exception");
table.table_id_ = "";
status = impl_->CreateTable(table);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.CreateTable.insert_throw_exception");
//success create table
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTable.throw_exception");
status = impl_->DescribeTable(table);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DescribeTable.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.HasTable.throw_exception");
bool has = false;
status = impl_->HasTable(table.table_id_, has);
ASSERT_FALSE(status.ok());
ASSERT_FALSE(has);
fiu_disable("SqliteMetaImpl.HasTable.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.AllTables.throw_exception");
std::vector<milvus::engine::meta::TableSchema> table_schema_array;
status = impl_->AllTables(table_schema_array);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.AllTables.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DropTable.throw_exception");
status = impl_->DropTable(table.table_id_);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DropTable.throw_exception");
}
{
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = "notexist";
status = impl_->CreateTableFile(schema);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.CreateTableFile.throw_exception");
schema.table_id_ = table_id;
status = impl_->CreateTableFile(schema);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.CreateTableFile.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DeleteTableFiles.throw_exception");
status = impl_->DeleteTableFiles(table.table_id_);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DeleteTableFiles.throw_exception");
}
{
milvus::engine::meta::DatesT dates;
status = impl_->DropDataByDate(table.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.push_back(1);
status = impl_->DropDataByDate("notexist", dates);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.DropDataByDate.throw_exception");
status = impl_->DropDataByDate(table.table_id_, dates);
ASSERT_FALSE(status.ok());
fiu_disable("SqliteMetaImpl.DropDataByDate.throw_exception");
}
{
milvus::engine::meta::TableFilesSchema schemas;
std::vector<size_t> ids;
status = impl_->GetTableFiles("notexist", ids, schemas);
ASSERT_FALSE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.GetTableFiles.throw_exception");
status = impl_->GetTableFiles(table_id, ids, schemas);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.GetTableFiles.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFlag.throw_exception");
status = impl_->UpdateTableFlag(table_id, 0);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFlag.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFile.throw_exception");
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = table_id;
status = impl_->UpdateTableFile(schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFile.throw_exception");
schema = {};
schema.table_id_ = "notexist";
status = impl_->UpdateTableFile(schema);
ASSERT_TRUE(status.ok());
}
{
milvus::engine::meta::TableFilesSchema schemas;
milvus::engine::meta::TableFileSchema schema;
schema.table_id_ = "notexits";
schemas.emplace_back(schema);
status = impl_->UpdateTableFiles(schemas);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFiles.throw_exception");
status = impl_->UpdateTableFiles(schemas);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFiles.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFiles.fail_commited");
status = impl_->UpdateTableFiles(schemas);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFiles.fail_commited");
}
{
milvus::engine::TableIndex index;
status = impl_->UpdateTableIndex("notexist", index);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableIndex.throw_exception");
status = impl_->UpdateTableIndex("notexist", index);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableIndex.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.DescribeTableIndex.throw_exception");
status = impl_->DescribeTableIndex(table_id, index);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.DescribeTableIndex.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception");
status = impl_->UpdateTableFilesToIndex(table_id);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.UpdateTableFilesToIndex.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.DropTableIndex.throw_exception");
status = impl_->DropTableIndex(table_id);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.DropTableIndex.throw_exception");
}
{
std::string partition = "part0";
std::string partition_tag = "tag0";
status = impl_->CreatePartition("notexist", partition, partition_tag);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_TRUE(status.ok());
partition_tag = "tag1";
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_FALSE(status.ok());
//create empty name partition
partition = "";
status = impl_->CreatePartition(table_id, partition, partition_tag);
ASSERT_TRUE(status.ok());
std::vector<milvus::engine::meta::TableSchema> partions_schema;
status = impl_->ShowPartitions(table_id, partions_schema);
ASSERT_TRUE(status.ok());
ASSERT_EQ(partions_schema.size(), 2);
partions_schema.clear();
FIU_ENABLE_FIU("SqliteMetaImpl.ShowPartitions.throw_exception");
status = impl_->ShowPartitions(table_id, partions_schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.ShowPartitions.throw_exception");
std::string partion;
FIU_ENABLE_FIU("SqliteMetaImpl.GetPartitionName.throw_exception");
status = impl_->GetPartitionName(table_id, "tag0", partion);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.GetPartitionName.throw_exception");
}
{
std::vector<size_t> ids;
milvus::engine::meta::DatesT dates;
milvus::engine::meta::DatePartionedTableFilesSchema schema;
status = impl_->FilesToSearch("notexist", ids, dates, schema);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToSearch.throw_exception");
status = impl_->FilesToSearch(table_id, ids, dates, schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToSearch.throw_exception");
}
{
milvus::engine::meta::TableFileSchema file;
file.table_id_ = table_id;
file.file_type_ = milvus::engine::meta::TableFileSchema::RAW;
status = impl_->CreateTableFile(file);
ASSERT_TRUE(status.ok());
file.file_size_ = std::numeric_limits<size_t>::max();
status = impl_->UpdateTableFile(file);
ASSERT_TRUE(status.ok());
milvus::engine::meta::DatePartionedTableFilesSchema schema;
status = impl_->FilesToMerge("notexist", schema);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToMerge.throw_exception");
status = impl_->FilesToMerge(table_id, schema);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToMerge.throw_exception");
//skip large files
milvus::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[file.date_].size(), 0);
}
{
milvus::engine::meta::TableFileSchema file;
file.table_id_ = table_id;
status = impl_->CreateTableFile(file);
ASSERT_TRUE(status.ok());
file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX;
impl_->UpdateTableFile(file);
milvus::engine::meta::TableFilesSchema files;
FIU_ENABLE_FIU("SqliteMetaImpl_FilesToIndex_TableNotFound");
status = impl_->FilesToIndex(files);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
fiu_disable("SqliteMetaImpl_FilesToIndex_TableNotFound");
FIU_ENABLE_FIU("SqliteMetaImpl.FilesToIndex.throw_exception");
status = impl_->FilesToIndex(files);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesToIndex.throw_exception");
}
{
milvus::engine::meta::TableFilesSchema files;
std::vector<int> file_types;
file_types.push_back(milvus::engine::meta::TableFileSchema::INDEX);
FIU_ENABLE_FIU("SqliteMetaImpl.FilesByType.throw_exception");
status = impl_->FilesByType(table_id, file_types, files);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.FilesByType.throw_exception");
}
{
uint64_t size = 0;
FIU_ENABLE_FIU("SqliteMetaImpl.Size.throw_exception");
status = impl_->Size(size);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Size.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpShadowFiles.fail_commited");
status = impl_->CleanUpShadowFiles();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpShadowFiles.fail_commited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpShadowFiles.throw_exception");
status = impl_->CleanUpShadowFiles();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpShadowFiles.throw_exception");
}
{
uint64_t count;
status = impl_->Count("notexist", count);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("SqliteMetaImpl.Count.throw_exception");
status = impl_->Count("notexist", count);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Count.throw_exception");
}
{
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_ThrowException");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveFile_FailCommited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_Failcommited");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTable_ThrowException");
FIU_ENABLE_FIU("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException");
status = impl_->CleanUpFilesWithTTL(1);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.CleanUpFilesWithTTL.RemoveTableFolder_ThrowException");
}
}
TEST_F(MetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
@ -148,6 +493,14 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
ids.push_back(table_file.id_);
}
{
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.Archive.throw_exception");
status = impl.Archive();
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
fiu_disable("SqliteMetaImpl.Archive.throw_exception");
}
impl.Archive();
int i = 0;
@ -193,6 +546,17 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
ids.push_back(table_file.id_);
}
{
fiu_init(0);
FIU_ENABLE_FIU("SqliteMetaImpl.DiscardFiles.throw_exception");
status = impl.Archive();
fiu_disable("SqliteMetaImpl.DiscardFiles.throw_exception");
FIU_ENABLE_FIU("SqliteMetaImpl.DiscardFiles.fail_commited");
status = impl.Archive();
fiu_disable("SqliteMetaImpl.DiscardFiles.fail_commited");
}
impl.Archive();
int i = 0;
@ -312,9 +676,9 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, table_files);
@ -337,6 +701,21 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
status = impl_->CleanUpFilesWithTTL(1UL);
ASSERT_TRUE(status.ok());
sleep(1);
std::vector<int> files_to_delete;
milvus::engine::meta::TableFilesSchema files_schema;
files_to_delete.push_back(milvus::engine::meta::TableFileSchema::TO_DELETE);
status = impl_->FilesByType(table_id, files_to_delete, files_schema);
ASSERT_TRUE(status.ok());
table_file.table_id_ = table_id;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_DELETE;
milvus::engine::OngoingFileChecker filter;
table_file.file_id_ = files_schema.front().file_id_;
filter.MarkOngoingFile(table_file);
status = impl_->CleanUpFilesWithTTL(1UL, &filter);
ASSERT_TRUE(status.ok());
}
TEST_F(MetaTest, INDEX_TEST) {

433
core/unittest/db/test_meta_mysql.cpp
View File

@ -24,11 +24,19 @@
#include <mysql++/mysql++.h>
#include <stdlib.h>
#include <time.h>
#include <boost/filesystem/operations.hpp>
#include <iostream>
#include <thread>
#include <fiu-local.h>
#include <fiu-control.h>
#include <src/db/OngoingFileChecker.h>
const char* FAILED_CONNECT_SQL_SERVER = "Failed to connect to meta server(mysql)";
const char* TABLE_ALREADY_EXISTS = "Table already exists and it is in delete state, please wait a second";
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
fiu_init(0);
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
@ -54,43 +62,203 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
status = impl_->CreateTable(table);
// ASSERT_TRUE(status.ok());
table.table_id_ = table_id;
FIU_ENABLE_FIU("MySQLMetaImpl.CreateTable.null_connection");
auto stat = impl_->CreateTable(table);
ASSERT_FALSE(stat.ok());
ASSERT_EQ(stat.message(), FAILED_CONNECT_SQL_SERVER);
fiu_disable("MySQLMetaImpl.CreateTable.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.CreateTable.throw_exception");
stat = impl_->CreateTable(table);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.CreateTable.throw_exception");
//ensure table exists
stat = impl_->CreateTable(table);
FIU_ENABLE_FIU("MySQLMetaImpl.CreateTableTable.schema_TO_DELETE");
stat = impl_->CreateTable(table);
ASSERT_FALSE(stat.ok());
ASSERT_EQ(stat.message(), TABLE_ALREADY_EXISTS);
fiu_disable("MySQLMetaImpl.CreateTableTable.schema_TO_DELETE");
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.null_connection");
stat = impl_->DescribeTable(table);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
stat = impl_->DescribeTable(table);
ASSERT_FALSE(stat.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
bool has_table = false;
stat = impl_->HasTable(table_id, has_table);
ASSERT_TRUE(stat.ok());
ASSERT_TRUE(has_table);
has_table = false;
FIU_ENABLE_FIU("MySQLMetaImpl.HasTable.null_connection");
stat = impl_->HasTable(table_id, has_table);
ASSERT_FALSE(stat.ok());
ASSERT_FALSE(has_table);
fiu_disable("MySQLMetaImpl.HasTable.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.HasTable.throw_exception");
stat = impl_->HasTable(table_id, has_table);
ASSERT_FALSE(stat.ok());
ASSERT_FALSE(has_table);
fiu_disable("MySQLMetaImpl.HasTable.throw_exception");
FIU_ENABLE_FIU("MySQLMetaImpl.DropTable.CLUSTER_WRITABLE_MODE");
stat = impl_->DropTable(table_id);
fiu_disable("MySQLMetaImpl.DropTable.CLUSTER_WRITABLE_MODE");
FIU_ENABLE_FIU("MySQLMetaImpl.DropAll.null_connection");
status = impl_->DropAll();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DropAll.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DropAll.throw_exception");
status = impl_->DropAll();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DropAll.throw_exception");
status = impl_->DropAll();
ASSERT_TRUE(status.ok());
}
TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
fiu_init(0);
uint64_t size = 0;
auto status = impl_->Size(size);
ASSERT_TRUE(status.ok());
ASSERT_EQ(size, 0);
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
status = impl_->CreateTable(table);
//CreateTableFile
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
FIU_ENABLE_FIU("MySQLMetaImpl.CreateTableFiles.null_connection");
status = impl_->CreateTableFile(table_file);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CreateTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.CreateTableFiles.throw_exception");
status = impl_->CreateTableFile(table_file);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CreateTableFiles.throw_exception");
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
status = impl_->CreateTableFile(table_file);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
//DropDataByDate
milvus::engine::meta::DatesT dates;
dates.push_back(milvus::engine::utils::GetDate());
dates.clear();
status = impl_->DropDataByDate(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.push_back(milvus::engine::utils::GetDate());
status = impl_->DropDataByDate("notexist", dates);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
FIU_ENABLE_FIU("MySQLMetaImpl.DropDataByDate.null_connection");
status = impl_->DropDataByDate(table_file.table_id_, dates);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DropDataByDate.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DropDataByDate.throw_exception");
status = impl_->DropDataByDate(table_file.table_id_, dates);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DropDataByDate.throw_exception");
status = impl_->DropDataByDate(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
//Count
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL);
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
status = impl_->Count(table_id, cnt);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
FIU_ENABLE_FIU("MySQLMetaImpl.Count.null_connection");
status = impl_->Count(table_id, cnt);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.Count.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.Count.throw_exception");
status = impl_->Count(table_id, cnt);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.Count.throw_exception");
auto file_id = table_file.file_id_;
auto new_file_type = milvus::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
//UpdateTableFile
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFile.null_connection");
status = impl_->UpdateTableFile(table_file);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFile.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFile.throw_exception");
status = impl_->UpdateTableFile(table_file);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFile.throw_exception");
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, new_file_type);
auto no_table_file = table_file;
no_table_file.table_id_ = "notexist";
status = impl_->UpdateTableFile(no_table_file);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpShadowFiles.null_connection");
status = impl_->CleanUpShadowFiles();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpShadowFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpShadowFiles.throw_exception");
status = impl_->CleanUpShadowFiles();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpShadowFiles.throw_exception");
status = impl_->CleanUpShadowFiles();
ASSERT_TRUE(status.ok());
milvus::engine::meta::TableFilesSchema files_schema;
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFiles.null_connection");
status = impl_->UpdateTableFiles(files_schema);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFiles.throw_exception");
status = impl_->UpdateTableFiles(files_schema);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFiles.throw_exception");
status = impl_->UpdateTableFiles(files_schema);
ASSERT_TRUE(status.ok());
dates.clear();
for (auto i = 2; i < 10; ++i) {
dates.push_back(milvus::engine::utils::GetDateWithDelta(-1 * i));
@ -113,9 +281,44 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
milvus::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_EQ(files.size(), 0UL);
FIU_ENABLE_FIU("MySQLMetaImpl.GetTableFiles.null_connection");
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.GetTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.GetTableFiles.throw_exception");
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.GetTableFiles.throw_exception");
ids.clear();
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
sleep(1);
std::vector<int> files_to_delete;
files_to_delete.push_back(milvus::engine::meta::TableFileSchema::TO_DELETE);
status = impl_->FilesByType(table_id, files_to_delete, files_schema);
ASSERT_TRUE(status.ok());
table_file.table_id_ = table_id;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_DELETE;
milvus::engine::OngoingFileChecker filter;
table_file.file_id_ = files_schema.front().file_id_;
filter.MarkOngoingFile(table_file);
status = impl_->CleanUpFilesWithTTL(1UL, &filter);
ASSERT_TRUE(status.ok());
status = impl_->DropTable(table_file.table_id_);
ASSERT_TRUE(status.ok());
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
fiu_init(0);
srand(time(0));
milvus::engine::DBMetaOptions options = GetOptions().meta_;
@ -152,6 +355,16 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
ids.push_back(table_file.id_);
}
FIU_ENABLE_FIU("MySQLMetaImpl.Archive.null_connection");
status = impl.Archive();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.Archive.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.Archive.throw_exception");
status = impl.Archive();
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.Archive.throw_exception");
impl.Archive();
int i = 0;
@ -173,14 +386,38 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
status = impl.FilesByType(table_id, file_types, table_files);
ASSERT_FALSE(table_files.empty());
FIU_ENABLE_FIU("MySQLMetaImpl.FilesByType.null_connection");
table_files.clear();
status = impl.FilesByType(table_id, file_types, table_files);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(table_files.empty());
fiu_disable("MySQLMetaImpl.FilesByType.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.FilesByType.throw_exception");
status = impl.FilesByType(table_id, file_types, table_files);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(table_files.empty());
fiu_disable("MySQLMetaImpl.FilesByType.throw_exception");
status = impl.UpdateTableFilesToIndex(table_id);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFilesToIndex.null_connection");
status = impl.UpdateTableFilesToIndex(table_id);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFilesToIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFilesToIndex.throw_exception");
status = impl.UpdateTableFilesToIndex(table_id);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFilesToIndex.throw_exception");
status = impl.DropAll();
ASSERT_TRUE(status.ok());
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
fiu_init(0);
milvus::engine::DBMetaOptions options = GetOptions().meta_;
options.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:11");
@ -212,6 +449,14 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
ids.push_back(table_file.id_);
}
FIU_ENABLE_FIU("MySQLMetaImpl.DiscardFiles.null_connection");
impl.Archive();
fiu_disable("MySQLMetaImpl.DiscardFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DiscardFiles.throw_exception");
impl.Archive();
fiu_disable("MySQLMetaImpl.DiscardFiles.throw_exception");
impl.Archive();
int i = 0;
@ -230,8 +475,57 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
ASSERT_TRUE(status.ok());
}
TEST_F(MySqlMetaTest, INVALID_INITILIZE_TEST) {
fiu_init(0);
auto table_id = "meta_test_group";
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
milvus::engine::DBMetaOptions meta = GetOptions().meta_;
{
FIU_ENABLE_FIU("MySQLMetaImpl.Initialize.fail_create_directory");
//delete directory created by SetUp
boost::filesystem::remove_all(meta.path_);
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(meta, GetOptions().mode_));
fiu_disable("MySQLMetaImpl.Initialize.fail_create_directory");
}
{
meta.backend_uri_ = "null";
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(meta, GetOptions().mode_));
}
{
meta.backend_uri_ = "notmysql://root:123456@127.0.0.1:3306/test";
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(meta, GetOptions().mode_));
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.Initialize.is_thread_aware");
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(GetOptions().meta_, GetOptions().mode_));
fiu_disable("MySQLMetaImpl.Initialize.is_thread_aware");
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.Initialize.fail_create_table_scheme");
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(GetOptions().meta_, GetOptions().mode_));
fiu_disable("MySQLMetaImpl.Initialize.fail_create_table_scheme");
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.Initialize.fail_create_table_files");
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(GetOptions().meta_, GetOptions().mode_));
fiu_disable("MySQLMetaImpl.Initialize.fail_create_table_files");
}
{
FIU_ENABLE_FIU("MySQLConnectionPool.create.throw_exception");
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(GetOptions().meta_, GetOptions().mode_));
fiu_disable("MySQLConnectionPool.create.throw_exception");
}
{
FIU_ENABLE_FIU("MySQLMetaImpl.ValidateMetaSchema.fail_validate");
ASSERT_ANY_THROW(milvus::engine::meta::MySQLMetaImpl impl(GetOptions().meta_, GetOptions().mode_));
fiu_disable("MySQLMetaImpl.ValidateMetaSchema.fail_validate");
}
}
TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
fiu_init(0);
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
@ -307,9 +601,49 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToMerge.null_connection");
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToMerge.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToMerge.throw_exception");
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToMerge.throw_exception");
status = impl_->FilesToMerge("notexist", dated_files);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
table_file.file_type_ = milvus::engine::meta::TableFileSchema::RAW;
table_file.file_size_ = milvus::engine::ONE_GB + 1;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
{
//skip large files
milvus::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
}
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTable.throw_exception");
status = impl_->FilesToIndex(files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DescribeTable.throw_exception");
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToIndex.null_connection");
status = impl_->FilesToIndex(files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToIndex.throw_exception");
status = impl_->FilesToIndex(files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToIndex.throw_exception");
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
@ -325,6 +659,19 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), 0);
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToSearch.null_connection");
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToSearch.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.FilesToSearch.throw_exception");
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.FilesToSearch.throw_exception");
status = impl_->FilesToSearch("notexist", ids, dates, dated_files);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
std::vector<int> file_types;
milvus::engine::meta::TableFilesSchema table_files;
status = impl_->FilesByType(table.table_id_, file_types, table_files);
@ -332,9 +679,9 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, table_files);
@ -343,6 +690,16 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(table_files.size(), total_cnt);
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.null_connection");
status = impl_->DeleteTableFiles(table_id);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DeleteTableFiles.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DeleteTableFiles.throw_exception");
status = impl_->DeleteTableFiles(table_id);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DeleteTableFiles.throw_exception");
status = impl_->DeleteTableFiles(table_id);
ASSERT_TRUE(status.ok());
@ -351,10 +708,41 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_NullConnection");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_NullConnection");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_ThrowException");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RomoveToDeleteFiles_ThrowException");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_NUllConnection");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_NUllConnection");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_ThrowException");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveToDeleteTables_ThrowException");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_NUllConnection");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_NUllConnection");
FIU_ENABLE_FIU("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_ThrowException");
status = impl_->CleanUpFilesWithTTL(0UL);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.CleanUpFilesWithTTL.RemoveDeletedTableFolder_ThrowException");
}
TEST_F(MySqlMetaTest, INDEX_TEST) {
auto table_id = "index_test";
fiu_init(0);
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
@ -367,10 +755,33 @@ TEST_F(MySqlMetaTest, INDEX_TEST) {
status = impl_->UpdateTableIndex(table_id, index);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableIndex.null_connection");
status = impl_->UpdateTableIndex(table_id, index);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableIndex.throw_exception");
status = impl_->UpdateTableIndex(table_id, index);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableIndex.throw_exception");
status = impl_->UpdateTableIndex("notexist", index);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
int64_t flag = 65536;
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFlag.null_connection");
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFlag.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.UpdateTableFlag.throw_exception");
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.UpdateTableFlag.throw_exception");
milvus::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
@ -389,6 +800,20 @@ TEST_F(MySqlMetaTest, INDEX_TEST) {
ASSERT_NE(index_out.nlist_, index.nlist_);
ASSERT_NE(index_out.engine_type_, index.engine_type_);
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTableIndex.null_connection");
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DescribeTableIndex.null_connection");
FIU_ENABLE_FIU("MySQLMetaImpl.DescribeTableIndex.throw_exception");
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_FALSE(status.ok());
fiu_disable("MySQLMetaImpl.DescribeTableIndex.throw_exception");
status = impl_->DescribeTableIndex("notexist", index_out);
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
status = impl_->UpdateTableFilesToIndex(table_id);
ASSERT_TRUE(status.ok());
}

42
core/unittest/db/test_misc.cpp
View File

@ -28,6 +28,9 @@
#include <boost/filesystem.hpp>
#include <thread>
#include <vector>
#include <fiu-local.h>
#include <fiu-control.h>
#include "db/utils.h"
TEST(DBMiscTest, EXCEPTION_TEST) {
milvus::Exception ex1(100, "error");
@ -91,7 +94,20 @@ TEST(DBMiscTest, UTILS_TEST) {
options.slave_paths_.push_back("/tmp/milvus_test/slave_2");
const std::string TABLE_NAME = "test_tbl";
auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
fiu_init(0);
milvus::Status status;
FIU_ENABLE_FIU("CommonUtil.CreateDirectory.create_parent_fail");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
FIU_ENABLE_FIU("CreateTablePath.creat_slave_path");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_FALSE(status.ok());
fiu_disable("CreateTablePath.creat_slave_path");
status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for (auto& path : options.slave_paths_) {
@ -120,6 +136,30 @@ TEST(DBMiscTest, UTILS_TEST) {
status = milvus::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
status = milvus::engine::utils::CreateTableFilePath(options, file);
ASSERT_TRUE(status.ok());
FIU_ENABLE_FIU("CreateTableFilePath.fail_create");
status = milvus::engine::utils::CreateTableFilePath(options, file);
ASSERT_FALSE(status.ok());
fiu_disable("CreateTableFilePath.fail_create");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
FIU_ENABLE_FIU("CommonUtil.CreateDirectory.create_parent_fail");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
FIU_ENABLE_FIU("GetTableFilePath.enable_s3");
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(file.location_.empty());
fiu_disable("GetTableFilePath.enable_s3");
status = milvus::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
}
TEST(DBMiscTest, CHECKER_TEST) {

3
core/unittest/db/utils.cpp
View File

@ -25,6 +25,7 @@
#include <string>
#include <thread>
#include <utility>
#include <fiu-local.h>
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
@ -231,7 +232,7 @@ MetaTest::TearDown() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
milvus::engine::DBOptions
MySqlDBTest::GetOptions() {
MySqlDBTest:: GetOptions() {
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = test_env->getURI();

6
core/unittest/db/utils.h
View File

@ -20,6 +20,8 @@
#include <gtest/gtest.h>
#include <chrono>
#include <memory>
#include <fiu-control.h>
#include <fiu-local.h>
#include "db/DB.h"
#include "db/meta/MySQLMetaImpl.h"
@ -44,6 +46,10 @@
#define STOP_TIMER(name)
#endif
#ifdef FIU_ENABLE
#define FIU_ENABLE_FIU(name) fiu_enable(name, 1, nullptr, 0)
#endif
static const char* CONFIG_PATH = "/tmp/milvus_test";
static const char* CONFIG_FILE = "/server_config.yaml";

2
core/unittest/main.cpp
View File

@ -17,6 +17,7 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <fiu-local.h>
#include "easyloggingpp/easylogging++.h"
@ -24,6 +25,7 @@ INITIALIZE_EASYLOGGINGPP
int
main(int argc, char** argv) {
fiu_init(0);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1
core/unittest/metrics/test_metricbase.cpp
View File

@ -56,6 +56,7 @@ TEST(MetricbaseTest, METRICBASE_TEST) {
instance.RAMUsagePercentSet();
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);

27
core/unittest/metrics/test_metrics.cpp
View File

@ -21,13 +21,17 @@
#include <string>
#include <thread>
#include <gtest/gtest.h>
#include <fiu-local.h>
#include <fiu-control.h>
#define private public
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "metrics/Metrics.h"
namespace {
static constexpr int64_t TABLE_DIM = 256;
@ -46,13 +50,30 @@ BuildVectors(uint64_t n, milvus::engine::VectorsData& vectors) {
} // namespace
TEST_F(MetricTest, METRIC_TEST) {
fiu_init(0);
#ifdef MILVUS_GPU_VERSION
FIU_ENABLE_FIU("SystemInfo.Init.nvmInit_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
milvus::server::SystemInfo::GetInstance().Init();
fiu_disable("SystemInfo.Init.nvmInit_fail");
FIU_ENABLE_FIU("SystemInfo.Init.nvm_getDevice_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
milvus::server::SystemInfo::GetInstance().Init();
fiu_disable("SystemInfo.Init.nvm_getDevice_fail");
milvus::server::SystemInfo::GetInstance().initialized_ = false;
#endif
milvus::server::SystemInfo::GetInstance().Init();
milvus::server::Metrics::GetInstance().Init();
std::string system_info;
milvus::server::SystemInfo::GetInstance().GetSysInfoJsonStr(system_info);
milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char *group_name = "test_group";
static const char* group_name = "test_group";
static const int group_dim = 256;
milvus::engine::meta::TableSchema group_info;
@ -90,7 +111,7 @@ TEST_F(MetricTest, METRIC_TEST) {
START_TIMER;
// stat = db_->Query(group_name, tags, k, qb, qxb, result_ids, result_distances);
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
ss << "Search " << j << " With Size " << (float)(count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
for (auto k = 0; k < qb; ++k) {

29
core/unittest/metrics/test_prometheus.cpp
View File

@ -20,8 +20,11 @@
#include <gtest/gtest.h>
#include <iostream>
#include <fiu-control.h>
#include <fiu-local.h>
TEST(PrometheusTest, PROMETHEUS_TEST) {
fiu_init(0);
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
milvus::server::PrometheusMetrics instance = milvus::server::PrometheusMetrics::GetInstance();
@ -57,9 +60,20 @@ TEST(PrometheusTest, PROMETHEUS_TEST) {
instance.QueryVectorResponseSummaryObserve(1.0, 1);
instance.QueryVectorResponsePerSecondGaugeSet(1.0);
instance.CPUUsagePercentSet();
fiu_enable("SystemInfo.CPUPercent.mock", 1, nullptr, 0);
instance.CPUUsagePercentSet();
fiu_disable("SystemInfo.CPUPercent.mock");
instance.RAMUsagePercentSet();
fiu_enable("SystemInfo.MemoryPercent.mock", 1, nullptr, 0);
instance.RAMUsagePercentSet();
fiu_disable("SystemInfo.MemoryPercent.mock");
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
fiu_enable("SystemInfo.GPUMemoryTotal.mock", 1, nullptr, 0);
fiu_enable("SystemInfo.GPUMemoryUsed.mock", 1, nullptr, 0);
instance.GPUPercentGaugeSet();
fiu_disable("SystemInfo.GPUMemoryTotal.mock");
fiu_disable("SystemInfo.GPUMemoryUsed.mock");
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
@ -71,8 +85,23 @@ TEST(PrometheusTest, PROMETHEUS_TEST) {
instance.OctetsSet();
instance.CPUCoreUsagePercentSet();
fiu_enable("SystemInfo.getTotalCpuTime.open_proc", 1, nullptr, 0);
instance.CPUCoreUsagePercentSet();
fiu_disable("SystemInfo.getTotalCpuTime.open_proc");
fiu_enable("SystemInfo.getTotalCpuTime.read_proc", 1, nullptr, 0);
instance.CPUCoreUsagePercentSet();
fiu_disable("SystemInfo.getTotalCpuTime.read_proc");
instance.GPUTemperature();
fiu_enable("SystemInfo.GPUTemperature.mock", 1, nullptr, 0);
instance.GPUTemperature();
fiu_disable("SystemInfo.GPUTemperature.mock");
instance.CPUTemperature();
fiu_enable("SystemInfo.CPUTemperature.opendir", 1, nullptr, 0);
instance.CPUTemperature();
fiu_disable("SystemInfo.CPUTemperature.opendir");
fiu_enable("SystemInfo.CPUTemperature.openfile", 1, nullptr, 0);
instance.CPUTemperature();
fiu_disable("SystemInfo.CPUTemperature.openfile");
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
instance.Init();

6
core/unittest/metrics/utils.h
View File

@ -24,7 +24,7 @@
#include "db/DB.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/meta/SqliteMetaImpl.h"
#include <fiu-control.h>
#define TIMING
#ifdef TIMING
@ -42,6 +42,10 @@
#define STOP_TIMER(name)
#endif
#ifdef FIU_ENABLE
#define FIU_ENABLE_FIU(name) fiu_enable(name, 1, nullptr, 0)
#endif
void
ASSERT_STATS(milvus::Status& stat);

2
core/unittest/scheduler/CMakeLists.txt
View File

@ -26,6 +26,8 @@ set(test_files
${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_scheduler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_task.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_job.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_optimizer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_tasktable.cpp)
add_executable(test_scheduler

40
core/unittest/scheduler/test_action.cpp Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
#include <src/scheduler/task/BuildIndexTask.h>
#include <src/scheduler/task/SearchTask.h>
#include <src/scheduler/optimizer/FaissIVFFlatPass.h>
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/optimizer/FaissFlatPass.h"
#include "scheduler/optimizer/FaissIVFPQPass.h"
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
#include "scheduler/optimizer/FaissIVFSQ8Pass.h"
namespace milvus {
namespace scheduler {
#ifdef MILVUS_GPU_VERSION
TEST(Action_Test, TESTACTION) {
}
#endif
} // namespace scheduler
} // namespace milvus

51
core/unittest/scheduler/test_job.cpp Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "scheduler/job/Job.h"
#include "scheduler/job/BuildIndexJob.h"
#include "scheduler/job/DeleteJob.h"
#include "scheduler/job/SearchJob.h"
namespace milvus {
namespace scheduler {
class TestJob : public Job {
public:
TestJob() : Job(JobType::INVALID) {}
};
TEST(JobTest, TestJob) {
engine::DBOptions options;
auto build_index_ptr = std::make_shared<BuildIndexJob>(nullptr, options);
build_index_ptr->Dump();
build_index_ptr->AddToIndexFiles(nullptr);
TestJob test_job;
test_job.Dump();
auto delete_ptr = std::make_shared<DeleteJob>("table_id", nullptr, 1);
delete_ptr->Dump();
engine::VectorsData vectors;
auto search_ptr = std::make_shared<SearchJob>(nullptr, 1, 1, vectors);
search_ptr->Dump();
search_ptr->AddIndexFile(nullptr);
}
} // namespace scheduler
} // namespace milvus

101
core/unittest/scheduler/test_optimizer.cpp Normal file
View File

@ -0,0 +1,101 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
#include "scheduler/task/BuildIndexTask.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/optimizer/FaissIVFFlatPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/resource/CpuResource.h"
#include "scheduler/optimizer/BuildIndexPass.h"
#include "scheduler/optimizer/FaissFlatPass.h"
#include "scheduler/optimizer/FaissIVFPQPass.h"
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
#include "scheduler/optimizer/FaissIVFSQ8Pass.h"
#include "scheduler/optimizer/FallbackPass.h"
namespace milvus {
namespace scheduler {
#ifdef MILVUS_GPU_VERSION
TEST(OptimizerTest, TEST_OPTIMIZER) {
BuildIndexPass build_index_pass;
fiu_init(0);
fiu_enable("BuildIndexPass.Init.get_config_fail", 1, NULL, 0);
ASSERT_ANY_THROW(build_index_pass.Init(););
fiu_disable("BuildIndexPass.Init.get_config_fail");
auto build_index_task = std::make_shared<XBuildIndexTask>(nullptr, nullptr);
fiu_enable("BuildIndexPass.Run.empty_gpu_ids", 1, NULL, 0);
ASSERT_FALSE(build_index_pass.Run(build_index_task));
fiu_disable("BuildIndexPass.Run.empty_gpu_ids");
FaissFlatPass faiss_flat_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_flat_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FaissIVFFlatPass faiss_ivf_flat_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_flat_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FaissIVFPQPass faiss_ivf_pq_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_pq_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
auto file = std::make_shared<TableFileSchema>();
file->engine_type_ = (int)engine::EngineType::FAISS_IVFFLAT;
auto search_task = std::make_shared<XSearchTask>(nullptr, file, nullptr);
ASSERT_FALSE(faiss_ivf_pq_pass.Run(search_task));
FaissIVFSQ8HPass faiss_ivf_q8h_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
faiss_ivf_q8h_pass.Init();
fiu_disable("check_config_gpu_search_threshold_fail");
auto search_task2 = std::make_shared<XSearchTask>(nullptr, file, nullptr);
ASSERT_FALSE(faiss_ivf_q8h_pass.Run(build_index_task));
ASSERT_FALSE(faiss_ivf_q8h_pass.Run(search_task2));
FaissIVFSQ8Pass faiss_ivf_q8_pass;
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
ASSERT_ANY_THROW(faiss_ivf_q8_pass.Init(););
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_disable("check_config_gpu_search_threshold_fail");
FallbackPass fall_back_pass;
fall_back_pass.Init();
auto task = std::make_shared<XBuildIndexTask>(nullptr, nullptr);
ResMgrInst::GetInstance()->Add(std::make_shared<CpuResource>("name1", 1, true));
fall_back_pass.Run(task);
}
#endif
} // namespace scheduler
} // namespace milvus

28
core/unittest/scheduler/test_resource.cpp
View File

@ -132,13 +132,17 @@ class ResourceAdvanceTest : public testing::Test {
void
WaitLoader(uint64_t count) {
std::unique_lock<std::mutex> lock(load_mutex_);
cv_.wait(lock, [&] { return load_count_ == count; });
cv_.wait(lock, [&] {
return load_count_ == count;
});
}
void
WaitExecutor(uint64_t count) {
std::unique_lock<std::mutex> lock(exec_mutex_);
cv_.wait(lock, [&] { return exec_count_ == count; });
cv_.wait(lock, [&] {
return exec_count_ == count;
});
}
ResourcePtr disk_resource_;
@ -207,6 +211,12 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) {
for (uint64_t i = 0; i < NUM; ++i) {
ASSERT_EQ(tasks[i]->exec_count_, 1);
}
std::stringstream out;
out << *std::static_pointer_cast<CpuResource>(cpu_resource_);
out << *std::static_pointer_cast<GpuResource>(gpu_resource_);
out << *std::static_pointer_cast<DiskResource>(disk_resource_);
out << *std::static_pointer_cast<TestResource>(test_resource_);
}
TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) {
@ -263,6 +273,20 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
for (uint64_t i = 0; i < NUM; ++i) {
ASSERT_EQ(tasks[i]->exec_count_, 1);
}
test_resource_->TaskAvgCost();
std::cout << test_resource_->Dump() << " " << test_resource_->NumOfTaskToExec() << std::endl;
auto null_resource = ResourceFactory::Create("invalid", "invalid", 0);
ASSERT_EQ(null_resource, nullptr);
}
TEST(Connection_Test, CONNECTION_TEST) {
std::string connection_name = "cpu";
uint64_t speed = 982;
Connection connection(connection_name, speed);
ASSERT_EQ(connection_name, connection.name());
ASSERT_EQ(speed, connection.speed());
std::cout << connection.Dump() << std::endl;
}
} // namespace scheduler

18
core/unittest/scheduler/test_resource_mgr.cpp
View File

@ -57,6 +57,11 @@ TEST_F(ResourceMgrBaseTest, ADD) {
auto resource = std::make_shared<TestResource>("test", 0, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
mgr1_->Start();
mgr1_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
mgr1_->Stop();
}
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
@ -85,6 +90,11 @@ TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Start();
mgr1_->Clear();
mgr1_->Stop();
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
ASSERT_EQ(mgr1_->GetNumOfResource(), 0);
}
@ -158,6 +168,10 @@ TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
ASSERT_FALSE(mgr1_->DumpTaskTables().empty());
}
TEST_F(ResourceMgrBaseTest, Start) {
empty_mgr_->Start();
}
/************ ResourceMgrAdvanceTest ************/
class ResourceMgrAdvanceTest : public testing::Test {
@ -184,7 +198,9 @@ class ResourceMgrAdvanceTest : public testing::Test {
TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
bool flag = false;
auto callback = [&](EventPtr event) { flag = true; };
auto callback = [&](EventPtr event) {
flag = true;
};
mgr1_->RegisterSubscriber(callback);
TableFileSchemaPtr dummy = nullptr;
disk_res->task_table().Put(

58
core/unittest/scheduler/test_scheduler.cpp
View File

@ -16,7 +16,10 @@
// under the License.
#include <gtest/gtest.h>
#include <fiu-local.h>
#include <fiu-control.h>
#include "src/scheduler/SchedInst.h"
#include "cache/DataObj.h"
#include "cache/GpuCacheMgr.h"
#include "scheduler/ResourceFactory.h"
@ -100,7 +103,7 @@ class SchedulerTest : public testing::Test {
SetUp() override {
res_mgr_ = std::make_shared<ResourceMgr>();
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, false);
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, false);
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true);
disk_resource_ = res_mgr_->Add(std::move(disk));
cpu_resource_ = res_mgr_->Add(std::move(cpu));
@ -212,5 +215,58 @@ class SchedulerTest2 : public testing::Test {
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
//}
TEST(SchedulerTestResource, SPECIFIED_RESOURCE_TEST) {
auto mock_index_ptr = std::make_shared<MockVecIndex>();
milvus::engine::Config config;
auto quantizer_ptr = mock_index_ptr->LoadQuantizer(config);
ASSERT_EQ(quantizer_ptr, nullptr);
auto vec_index_ptr = mock_index_ptr->LoadData(quantizer_ptr, config);
ASSERT_EQ(vec_index_ptr, nullptr);
auto s = mock_index_ptr->SetQuantizer(quantizer_ptr);
ASSERT_TRUE(s.ok());
s = mock_index_ptr->UnsetQuantizer();
ASSERT_TRUE(s.ok());
auto res = mock_index_ptr->CopyToGpuWithQuantizer(0, config);
ASSERT_EQ(res.first, nullptr);
ASSERT_EQ(res.second, nullptr);
using IndexType = milvus::engine::IndexType;
auto index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU, config);
ASSERT_EQ(index->GetType(), IndexType::SPTAG_KDT_RNT_CPU);
index = GetVecIndexFactory(IndexType::SPTAG_BKT_RNT_CPU, config);
ASSERT_EQ(index->GetType(), IndexType::SPTAG_BKT_RNT_CPU);
#ifdef MILVUS_GPU_VERSION
index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_GPU, config);
ASSERT_EQ(index->GetType(), IndexType::FAISS_IVFPQ_GPU);
#endif
index = GetVecIndexFactory(IndexType::NSG_MIX, config);
ASSERT_EQ(index->GetType(), IndexType::NSG_MIX);
index = GetVecIndexFactory(IndexType::INVALID, config);
ASSERT_EQ(index, nullptr);
knowhere::BinarySet empty_set;
auto res_ptr = LoadVecIndex(IndexType::INVALID, empty_set, 0);
ASSERT_EQ(res_ptr, nullptr);
}
TEST_F(SchedulerTest, schedule) {
scheduler_->Dump();
}
TEST(SchedulerService, service) {
fiu_enable("load_simple_config_mock", 1, nullptr, 0);
StartSchedulerService();
StopSchedulerService();
fiu_disable("load_simple_config_mock");
}
} // namespace scheduler
} // namespace milvus

152
core/unittest/scheduler/test_task.cpp
View File

@ -15,11 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
#include <opentracing/mocktracer/tracer.h>
#include <src/scheduler/SchedInst.h>
#include <src/scheduler/resource/CpuResource.h>
#include "db/meta/SqliteMetaImpl.h"
#include "db/DBFactory.h"
#include "scheduler/tasklabel/BroadcastLabel.h"
#include "scheduler/task/BuildIndexTask.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/task/TestTask.h"
namespace milvus {
namespace scheduler {
@ -44,5 +52,149 @@ TEST(TaskTest, INVALID_INDEX) {
build_task->Execute();
}
TEST(TaskTest, TEST_TASK) {
auto dummy_context = std::make_shared<milvus::server::Context>("dummy_request_id");
auto file = std::make_shared<TableFileSchema>();
auto label = std::make_shared<BroadcastLabel>();
TestTask task(dummy_context, file, label);
task.Load(LoadType::CPU2GPU, 0);
auto th = std::thread([&]() {
task.Execute();
});
task.Wait();
if (th.joinable()) {
th.join();
}
static const char* CONFIG_PATH = "/tmp/milvus_test";
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = CONFIG_PATH;
options.meta_.backend_uri_ = "sqlite://:@:/";
options.insert_cache_immediately_ = true;
auto meta_ptr = std::make_shared<milvus::engine::meta::SqliteMetaImpl>(options.meta_);
file->table_id_ = "111";
file->location_ = "/tmp/milvus_test/index_file1.txt";
auto build_index_job = std::make_shared<BuildIndexJob>(meta_ptr, options);
XBuildIndexTask build_index_task(file, label);
build_index_task.job_ = build_index_job;
build_index_task.Load(LoadType::TEST, 0);
fiu_init(0);
fiu_enable("XBuildIndexTask.Load.throw_std_exception", 1, NULL, 0);
build_index_task.Load(LoadType::TEST, 0);
fiu_disable("XBuildIndexTask.Load.throw_std_exception");
fiu_enable("XBuildIndexTask.Load.out_of_memory", 1, NULL, 0);
build_index_task.Load(LoadType::TEST, 0);
fiu_disable("XBuildIndexTask.Load.out_of_memory");
build_index_task.Execute();
// always enable 'create_table_success'
fiu_enable("XBuildIndexTask.Execute.create_table_success", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_enable("XBuildIndexTask.Execute.build_index_fail", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.build_index_fail");
// always enable 'has_table'
fiu_enable("XBuildIndexTask.Execute.has_table", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_enable("XBuildIndexTask.Execute.throw_std_exception", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
// always enable 'save_index_file_success'
fiu_enable("XBuildIndexTask.Execute.save_index_file_success", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_enable("XBuildIndexTask.Execute.update_table_file_fail", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, file->nlist_);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.update_table_file_fail");
fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
fiu_disable("XBuildIndexTask.Execute.has_table");
fiu_disable("XBuildIndexTask.Execute.create_table_success");
build_index_task.Execute();
// search task
engine::VectorsData vector;
auto search_job = std::make_shared<SearchJob>(dummy_context, 1, 1, vector);
file->metric_type_ = static_cast<int>(MetricType::IP);
file->engine_type_ = static_cast<int>(engine::EngineType::FAISS_IVFSQ8H);
opentracing::mocktracer::MockTracerOptions tracer_options;
auto mock_tracer =
std::shared_ptr<opentracing::Tracer>{new opentracing::mocktracer::MockTracer{std::move(tracer_options)}};
auto mock_span = mock_tracer->StartSpan("mock_span");
auto trace_context = std::make_shared<milvus::tracing::TraceContext>(mock_span);
dummy_context->SetTraceContext(trace_context);
XSearchTask search_task(dummy_context, file, label);
search_task.job_ = search_job;
std::string cpu_resouce_name = "cpu_name1";
std::vector<std::string> path = {cpu_resouce_name};
search_task.task_path_ = Path(path, 0);
ResMgrInst::GetInstance()->Add(std::make_shared<CpuResource>(cpu_resouce_name, 1, true));
search_task.Load(LoadType::CPU2GPU, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_enable("XSearchTask.Load.throw_std_exception", 1, NULL, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_disable("XSearchTask.Load.throw_std_exception");
fiu_enable("XSearchTask.Load.out_of_memory", 1, NULL, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_disable("XSearchTask.Load.out_of_memory");
fiu_enable("XSearchTask.Execute.search_fail", 1, NULL, 0);
search_task.Execute();
fiu_disable("XSearchTask.Execute.search_fail");
fiu_enable("XSearchTask.Execute.throw_std_exception", 1, NULL, 0);
search_task.Execute();
fiu_disable("XSearchTask.Execute.throw_std_exception");
search_task.Execute();
scheduler::ResultIds ids, tar_ids;
scheduler::ResultDistances distances, tar_distances;
XSearchTask::MergeTopkToResultSet(ids, distances, 1, 1, 1, true, tar_ids, tar_distances);
}
TEST(TaskTest, TEST_PATH) {
Path path;
auto empty_path = path.Current();
ASSERT_TRUE(empty_path.empty());
empty_path = path.Next();
ASSERT_TRUE(empty_path.empty());
empty_path = path.Last();
ASSERT_TRUE(empty_path.empty());
}
} // namespace scheduler
} // namespace milvus

49
core/unittest/server/test_cache.cpp
View File

@ -16,11 +16,14 @@
// under the License.
#include <gtest/gtest.h>
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include <fiu-control.h>
#include <fiu-local.h>
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
namespace {
class InvalidCacheMgr : public milvus::cache::CacheMgr<milvus::cache::DataObjPtr> {
@ -140,7 +143,7 @@ TEST(CacheTest, CPU_CACHE_TEST) {
ASSERT_EQ(cpu_mgr->CacheCapacity(), cap);
uint64_t item_count = 20;
for (uint64_t i = 0; i < item_count; i++) {
for (uint64_t i = 0; i < item_count + 1; i++) {
// each vector is 1k byte, total size less than 1G
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
@ -148,6 +151,10 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
//insert null data
std::shared_ptr<milvus::cache::DataObj> null_data_obj = nullptr;
cpu_mgr->InsertItem("index_null", null_data_obj);
auto obj = cpu_mgr->GetIndex("index_0");
ASSERT_TRUE(obj == nullptr);
@ -170,9 +177,20 @@ TEST(CacheTest, CPU_CACHE_TEST) {
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_6g", data_obj);
ASSERT_TRUE(cpu_mgr->ItemExists("index_6g"));
//insert aleady existed key
cpu_mgr->InsertItem("index_6g", data_obj);
}
cpu_mgr->PrintInfo();
cpu_mgr->ClearCache();
ASSERT_EQ(cpu_mgr->ItemCount(), 0);
// fiu_enable("CpuCacheMgr_CpuCacheMgr_ZeroCpucacheThreshold", 1, nullptr, 0);
// auto* cpu_cache_mgr = new milvus::cache::CpuCacheMgr();
// fiu_disable("CpuCacheMgr_CpuCacheMgr_ZeroCpucacheThreshold");
// delete cpu_cache_mgr;
}
#ifdef MILVUS_GPU_VERSION
@ -202,6 +220,11 @@ TEST(CacheTest, GPU_CACHE_TEST) {
gpu_mgr->ClearCache();
ASSERT_EQ(gpu_mgr->ItemCount(), 0);
// fiu_enable("GpuCacheMgr_GpuCacheMgr_ZeroGpucacheThreshold", 1, nullptr, 0);
// auto* gpu_cache_mgr = new milvus::cache::GpuCacheMgr();
// fiu_disable("GpuCacheMgr_GpuCacheMgr_ZeroGpucacheThreshold");
// delete gpu_cache_mgr;
}
#endif
@ -233,3 +256,23 @@ TEST(CacheTest, INVALID_TEST) {
ASSERT_EQ(mgr.GetItem("index_0"), nullptr);
}
}
TEST(CacheTest, PARTIAL_LRU_TEST) {
constexpr int MAX_SIZE = 5;
milvus::cache::LRU<int, int> lru(MAX_SIZE);
lru.put(0, 2);
lru.put(0, 3);
ASSERT_EQ(lru.size(), 1);
for (int i = 1; i < MAX_SIZE; ++i) {
lru.put(i, 0);
}
ASSERT_EQ(lru.size(), MAX_SIZE);
lru.put(99, 0);
ASSERT_EQ(lru.size(), MAX_SIZE);
ASSERT_TRUE(lru.exists(99));
ASSERT_ANY_THROW(lru.get(-1));
}

407
core/unittest/server/test_config.cpp
View File

@ -18,7 +18,6 @@
#include <gtest/gtest-death-test.h>
#include <gtest/gtest.h>
#include <cmath>
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "server/utils.h"
@ -27,6 +26,8 @@
#include "utils/ValidationUtil.h"
#include <limits>
#include <fiu-local.h>
#include <fiu-control.h>
namespace {
@ -52,6 +53,7 @@ TEST_F(ConfigTest, CONFIG_TEST) {
ASSERT_TRUE(s.ok());
config_mgr->Print();
config_mgr->DumpString();
milvus::server::ConfigNode& root_config = config_mgr->GetRootNode();
milvus::server::ConfigNode& server_config = root_config.GetChild("server_config");
@ -59,9 +61,26 @@ TEST_F(ConfigTest, CONFIG_TEST) {
milvus::server::ConfigNode& metric_config = root_config.GetChild("metric_config");
milvus::server::ConfigNode& cache_config = root_config.GetChild("cache_config");
milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
const auto& im_config_mgr = *static_cast<milvus::server::YamlConfigMgr*>(config_mgr);
const milvus::server::ConfigNode& im_root_config = im_config_mgr.GetRootNode();
const milvus::server::ConfigNode& im_invalid_config = im_root_config.GetChild("invalid_config");
const milvus::server::ConfigNode& im_not_exit_config = im_root_config.GetChild("not_exit_config");
ASSERT_EQ(im_not_exit_config.GetConfig().size(), 0);
ASSERT_EQ(im_root_config.DumpString(), root_config.DumpString());
ASSERT_EQ(im_invalid_config.DumpString(), invalid_config.DumpString());
auto valus = invalid_config.GetSequence("not_exist");
float ff = invalid_config.GetFloatValue("not_exist", 3.0);
ASSERT_EQ(ff, 3.0);
double not_exit_double = server_config.GetDoubleValue("not_exit", 3.0);
ASSERT_EQ(not_exit_double, 3.0);
int64_t not_exit_int64 = server_config.GetInt64Value("not_exit", 3);
ASSERT_EQ(not_exit_int64, 3);
int64_t not_exit_int32 = server_config.GetInt32Value("not_exit", 3);
ASSERT_EQ(not_exit_int32, 3);
bool not_exit_bool = server_config.GetBoolValue("not_exit", false);
ASSERT_FALSE(not_exit_bool);
std::string address = server_config.GetValue("address");
ASSERT_TRUE(!address.empty());
@ -104,6 +123,8 @@ TEST_F(ConfigTest, CONFIG_TEST) {
auto seq = server_config.GetSequence("seq");
ASSERT_EQ(seq.size(), 2UL);
server_config.SetValue("fake", "fake");
server_config.AddChild("fake", fake);
milvus::server::ConfigNode combine;
combine.Combine(server_config);
@ -600,3 +621,387 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
ASSERT_TRUE(config.ResetDefaultConfig().ok());
}
TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
fiu_init(0);
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);
ASSERT_TRUE(s.ok());
fiu_enable("check_config_version_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_version_fail");
/* server config */
fiu_enable("check_config_address_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_address_fail");
fiu_enable("check_config_port_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_port_fail");
fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_deploy_mode_fail");
fiu_enable("check_config_time_zone_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_time_zone_fail");
/* db config */
fiu_enable("check_config_primary_path_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_primary_path_fail");
fiu_enable("check_config_secondary_path_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_secondary_path_fail");
fiu_enable("check_config_backend_url_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_backend_url_fail");
fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_archive_disk_threshold_fail");
fiu_enable("check_config_archive_days_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_archive_days_threshold_fail");
fiu_enable("check_config_insert_buffer_size_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_insert_buffer_size_fail");
/* metric config */
fiu_enable("check_config_enable_monitor_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_enable_monitor_fail");
/* cache config */
fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_capacity_fail");
fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_threshold_fail");
fiu_enable("check_config_cache_insert_data_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cache_insert_data_fail");
/* engine config */
fiu_enable("check_config_use_blas_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_use_blas_threshold_fail");
fiu_enable("check_config_omp_thread_num_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_omp_thread_num_fail");
#ifdef MILVUS_GPU_VERSION
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_search_threshold_fail");
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_cache_capacity_fail");
fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_cache_threshold_fail");
fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_search_fail");
fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_build_index_fail");
#endif
fiu_enable("get_config_json_config_path_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("get_config_json_config_path_fail");
s = config.ValidateConfig();
ASSERT_TRUE(s.ok());
#ifdef MILVUS_GPU_VERSION
std::vector<int64_t> empty_value;
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
empty_value.clear();
s = config.GetGpuResourceConfigBuildIndexResources(empty_value);
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("get_gpu_config_build_index_resources.disable_gpu_resource_fail", 1, NULL, 0);
empty_value.clear();
s = config.GetGpuResourceConfigBuildIndexResources(empty_value);
ASSERT_FALSE(s.ok());
fiu_disable("get_gpu_config_build_index_resources.disable_gpu_resource_fail");
fiu_enable("get_gpu_config_search_resources.disable_gpu_resource_fail", 1, NULL, 0);
empty_value.clear();
s = config.GetGpuResourceConfigSearchResources(empty_value);
ASSERT_FALSE(s.ok());
fiu_disable("get_gpu_config_search_resources.disable_gpu_resource_fail");
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
empty_value.clear();
s = config.GetGpuResourceConfigSearchResources(empty_value);
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
int64_t value;
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("Config.GetGpuResourceConfigCacheCapacity.diable_gpu_resource");
fiu_enable("ValidationUtil.GetGpuMemory.return_error", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("ValidationUtil.GetGpuMemory.return_error");
fiu_enable("check_config_insert_buffer_size_fail", 1, NULL, 0);
s = config.GetCacheConfigCpuCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("check_config_insert_buffer_size_fail");
fiu_enable("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer", 1, NULL, 0);
s = config.GetCacheConfigCpuCacheCapacity(value);
ASSERT_FALSE(s.ok());
fiu_disable("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer");
float f_value;
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheThreshold(f_value);
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource", 1, NULL, 0);
s = config.GetGpuResourceConfigCacheThreshold(f_value);
ASSERT_FALSE(s.ok());
fiu_disable("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource");
#endif
}
TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
fiu_init(0);
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);
ASSERT_TRUE(s.ok());
s = config.ValidateConfig();
ASSERT_TRUE(s.ok());
/* server config */
fiu_enable("check_config_address_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_address_fail");
fiu_enable("check_config_port_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_port_fail");
fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_deploy_mode_fail");
fiu_enable("check_config_time_zone_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_time_zone_fail");
/* db config */
fiu_enable("check_config_primary_path_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_primary_path_fail");
fiu_enable("check_config_secondary_path_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_secondary_path_fail");
fiu_enable("check_config_backend_url_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_backend_url_fail");
fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_archive_disk_threshold_fail");
fiu_enable("check_config_archive_days_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_archive_days_threshold_fail");
fiu_enable("check_config_insert_buffer_size_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_insert_buffer_size_fail");
/* metric config */
fiu_enable("check_config_enable_monitor_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_enable_monitor_fail");
/* cache config */
fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_capacity_fail");
fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_threshold_fail");
fiu_enable("check_config_cache_insert_data_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cache_insert_data_fail");
/* engine config */
fiu_enable("check_config_use_blas_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_use_blas_threshold_fail");
fiu_enable("check_config_omp_thread_num_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_omp_thread_num_fail");
#ifdef MILVUS_GPU_VERSION
fiu_enable("check_config_gpu_search_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_search_threshold_fail");
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_cache_capacity_fail");
fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_cache_threshold_fail");
fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_search_fail");
fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_build_index_fail");
#endif
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());
}
TEST_F(ConfigTest, SERVER_CONFIG_OTHER_CONFIGS_FAIL_TEST) {
fiu_init(0);
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s;
std::string get_cmd, set_cmd;
std::string result, dummy;
s = config.ProcessConfigCli(result, "get_config");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(result, "get_config");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(result, "get_config 1");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(result, "get_config 1.1");
ASSERT_FALSE(s.ok());
std::string build_index_resources = "gpu0";
set_cmd =
gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
std::cout << set_cmd << std::endl;
s = config.ProcessConfigCli(dummy, set_cmd + " invalid");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(dummy, "set_config gpu_resource_config.build_index_resources.sss gpu0");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(dummy, "set_config gpu_resource_config_invalid.build_index_resources_invalid gpu0");
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(dummy, "invalid_config gpu_resource_config.build_index_resources.sss gpu0");
ASSERT_FALSE(s.ok());
#ifndef MILVUS_GPU_VERSION
s = config.ProcessConfigCli(dummy, gen_set_command(ms::CONFIG_TRACING,
ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources));
ASSERT_FALSE(s.ok());
#endif
}

426
core/unittest/server/test_rpc.cpp
View File

@ -35,6 +35,10 @@
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "utils/CommonUtil.h"
#include "server/grpc_impl/GrpcServer.h"
#include <fiu-local.h>
#include <fiu-control.h>
namespace {
@ -108,6 +112,7 @@ class RpcHandlerTest : public testing::Test {
request.set_index_file_size(INDEX_FILE_SIZE);
request.set_metric_type(1);
handler->SetContext(&context, dummy_context);
handler->random_id();
::grpc::Status grpc_status = handler->CreateTable(&context, &request, &status);
}
@ -172,6 +177,17 @@ TEST_F(RpcHandlerTest, HAS_TABLE_TEST) {
ASSERT_TRUE(status.error_code() == ::grpc::Status::OK.error_code());
int error_code = reply.status().error_code();
ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
fiu_init(0);
fiu_enable("HasTableRequest.OnExecute.table_not_exist", 1, NULL, 0);
handler->HasTable(&context, &request, &reply);
ASSERT_NE(reply.status().error_code(), ::milvus::grpc::ErrorCode::SUCCESS);
fiu_disable("HasTableRequest.OnExecute.table_not_exist");
fiu_enable("HasTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->HasTable(&context, &request, &reply);
ASSERT_NE(reply.status().error_code(), ::milvus::grpc::ErrorCode::SUCCESS);
fiu_disable("HasTableRequest.OnExecute.throw_std_exception");
}
TEST_F(RpcHandlerTest, INDEX_TEST) {
@ -196,6 +212,30 @@ TEST_F(RpcHandlerTest, INDEX_TEST) {
int error_code = response.error_code();
// ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
fiu_init(0);
fiu_enable("CreateIndexRequest.OnExecute.not_has_table", 1, NULL, 0);
grpc_status = handler->CreateIndex(&context, &request, &response);
ASSERT_TRUE(grpc_status.ok());
fiu_disable("CreateIndexRequest.OnExecute.not_has_table");
fiu_enable("CreateIndexRequest.OnExecute.throw_std.exception", 1, NULL, 0);
grpc_status = handler->CreateIndex(&context, &request, &response);
ASSERT_TRUE(grpc_status.ok());
fiu_disable("CreateIndexRequest.OnExecute.throw_std.exception");
fiu_enable("CreateIndexRequest.OnExecute.create_index_fail", 1, NULL, 0);
grpc_status = handler->CreateIndex(&context, &request, &response);
ASSERT_TRUE(grpc_status.ok());
fiu_disable("CreateIndexRequest.OnExecute.create_index_fail");
#ifdef MILVUS_GPU_VERSION
request.mutable_index()->set_index_type(static_cast<int>(milvus::engine::EngineType::FAISS_PQ));
fiu_enable("CreateIndexRequest.OnExecute.ip_meteric", 1, NULL, 0);
grpc_status = handler->CreateIndex(&context, &request, &response);
ASSERT_TRUE(grpc_status.ok());
fiu_disable("CreateIndexRequest.OnExecute.ip_meteric");
#endif
::milvus::grpc::TableName table_name;
::milvus::grpc::IndexParam index_param;
handler->DescribeIndex(&context, &table_name, &index_param);
@ -203,12 +243,33 @@ TEST_F(RpcHandlerTest, INDEX_TEST) {
handler->DescribeIndex(&context, &table_name, &index_param);
table_name.set_table_name(TABLE_NAME);
handler->DescribeIndex(&context, &table_name, &index_param);
fiu_init(0);
fiu_enable("DescribeIndexRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->DescribeIndex(&context, &table_name, &index_param);
fiu_disable("DescribeIndexRequest.OnExecute.throw_std_exception");
::milvus::grpc::Status status;
table_name.Clear();
handler->DropIndex(&context, &table_name, &status);
table_name.set_table_name("test5");
handler->DropIndex(&context, &table_name, &status);
table_name.set_table_name(TABLE_NAME);
fiu_init(0);
fiu_enable("DropIndexRequest.OnExecute.table_not_exist", 1, NULL, 0);
handler->DropIndex(&context, &table_name, &status);
fiu_disable("DropIndexRequest.OnExecute.table_not_exist");
fiu_enable("DropIndexRequest.OnExecute.drop_index_fail", 1, NULL, 0);
handler->DropIndex(&context, &table_name, &status);
fiu_disable("DropIndexRequest.OnExecute.drop_index_fail");
fiu_enable("DropIndexRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->DropIndex(&context, &table_name, &status);
fiu_disable("DropIndexRequest.OnExecute.throw_std_exception");
handler->DropIndex(&context, &table_name, &status);
}
@ -229,6 +290,49 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
}
handler->Insert(&context, &request, &vector_ids);
ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_init(0);
fiu_enable("InsertRequest.OnExecute.id_array_error", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.id_array_error");
fiu_enable("InsertRequest.OnExecute.db_not_found", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.db_not_found");
fiu_enable("InsertRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.describe_table_fail");
fiu_enable("InsertRequest.OnExecute.illegal_vector_id", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.illegal_vector_id");
fiu_enable("InsertRequest.OnExecute.illegal_vector_id2", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.illegal_vector_id2");
fiu_enable("InsertRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.throw_std_exception");
fiu_enable("InsertRequest.OnExecute.invalid_dim", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
ASSERT_NE(vector_ids.vector_id_array_size(), VECTOR_COUNT);
fiu_disable("InsertRequest.OnExecute.invalid_dim");
fiu_enable("InsertRequest.OnExecute.insert_fail", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
fiu_disable("InsertRequest.OnExecute.insert_fail");
fiu_enable("InsertRequest.OnExecute.invalid_ids_size", 1, NULL, 0);
handler->Insert(&context, &request, &vector_ids);
fiu_disable("InsertRequest.OnExecute.invalid_ids_size");
// insert vectors with wrong dim
std::vector<float> record_wrong_dim(TABLE_DIM - 1, 0.5f);
@ -297,6 +401,39 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
request.set_table_name(TABLE_NAME);
handler->Search(&context, &request, &response);
fiu_init(0);
fiu_enable("SearchRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.describe_table_fail");
fiu_enable("SearchRequest.OnExecute.invalod_rowrecord_array", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.invalod_rowrecord_array");
fiu_enable("SearchRequest.OnExecute.invalid_dim", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.invalid_dim");
fiu_enable("SearchRequest.OnExecute.invalid_partition_tags", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.invalid_partition_tags");
fiu_enable("SearchRequest.OnExecute.query_fail", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.query_fail");
fiu_enable("SearchRequest.OnExecute.empty_result_ids", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.empty_result_ids");
fiu_enable("SearchRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("SearchRequest.OnExecute.throw_std_exception");
fiu_enable("GrpcRequestHandler.Search.not_empty_file_ids", 1, NULL, 0);
handler->Search(&context, &request, &response);
fiu_disable("GrpcRequestHandler.Search.not_empty_file_ids");
::milvus::grpc::SearchInFilesParam search_in_files_param;
std::string* file_id = search_in_files_param.add_file_id_array();
*file_id = "test_tbl";
@ -339,6 +476,15 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
::grpc::Status status = handler->DescribeTable(&context, &table_name, &table_schema);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
fiu_init(0);
fiu_enable("DescribeTableRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->DescribeTable(&context, &table_name, &table_schema);
fiu_disable("DescribeTableRequest.OnExecute.describe_table_fail");
fiu_enable("DescribeTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->DescribeTable(&context, &table_name, &table_schema);
fiu_disable("DescribeTableRequest.OnExecute.throw_std_exception");
::milvus::grpc::InsertParam request;
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
@ -380,6 +526,11 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
status = handler->ShowTables(&context, &cmd, &table_name_list);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
fiu_init(0);
fiu_enable("ShowTablesRequest.OnExecute.show_tables_fail", 1, NULL, 0);
handler->ShowTables(&context, &cmd, &table_name_list);
fiu_disable("ShowTablesRequest.OnExecute.show_tables_fail");
// Count Table
::milvus::grpc::TableRowCount count;
table_name.Clear();
@ -388,6 +539,18 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
status = handler->CountTable(&context, &table_name, &count);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
// ASSERT_EQ(count.table_row_count(), vector_ids.vector_id_array_size());
fiu_init(0);
fiu_enable("CountTableRequest.OnExecute.db_not_found", 1, NULL, 0);
status = handler->CountTable(&context, &table_name, &count);
fiu_disable("CountTableRequest.OnExecute.db_not_found");
fiu_enable("CountTableRequest.OnExecute.status_error", 1, NULL, 0);
status = handler->CountTable(&context, &table_name, &count);
fiu_disable("CountTableRequest.OnExecute.status_error");
fiu_enable("CountTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
status = handler->CountTable(&context, &table_name, &count);
fiu_disable("CountTableRequest.OnExecute.throw_std_exception");
// Preload Table
table_name.Clear();
@ -396,15 +559,77 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
status = handler->PreloadTable(&context, &table_name, &response);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
fiu_enable("PreloadTableRequest.OnExecute.preload_table_fail", 1, NULL, 0);
handler->PreloadTable(&context, &table_name, &response);
fiu_disable("PreloadTableRequest.OnExecute.preload_table_fail");
fiu_enable("PreloadTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->PreloadTable(&context, &table_name, &response);
fiu_disable("PreloadTableRequest.OnExecute.throw_std_exception");
fiu_init(0);
fiu_enable("CreateTableRequest.OnExecute.invalid_index_file_size", 1, NULL, 0);
tableschema.set_table_name(tablename);
handler->CreateTable(&context, &tableschema, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreateTableRequest.OnExecute.invalid_index_file_size");
fiu_enable("CreateTableRequest.OnExecute.db_already_exist", 1, NULL, 0);
tableschema.set_table_name(tablename);
handler->CreateTable(&context, &tableschema, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreateTableRequest.OnExecute.db_already_exist");
fiu_enable("CreateTableRequest.OnExecute.create_table_fail", 1, NULL, 0);
tableschema.set_table_name(tablename);
handler->CreateTable(&context, &tableschema, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreateTableRequest.OnExecute.create_table_fail");
fiu_enable("CreateTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
tableschema.set_table_name(tablename);
handler->CreateTable(&context, &tableschema, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreateTableRequest.OnExecute.throw_std_exception");
// Drop table
table_name.set_table_name("");
// test invalid table name
::grpc::Status grpc_status = handler->DropTable(&context, &table_name, &response);
table_name.set_table_name(tablename);
fiu_enable("DropTableRequest.OnExecute.db_not_found", 1, NULL, 0);
handler->DropTable(&context, &table_name, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropTableRequest.OnExecute.db_not_found");
fiu_enable("DropTableRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->DropTable(&context, &table_name, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropTableRequest.OnExecute.describe_table_fail");
fiu_enable("DropTableRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->DropTable(&context, &table_name, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropTableRequest.OnExecute.throw_std_exception");
grpc_status = handler->DropTable(&context, &table_name, &response);
ASSERT_EQ(grpc_status.error_code(), ::grpc::Status::OK.error_code());
int error_code = status.error_code();
int error_code = response.error_code();
ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS);
tableschema.set_table_name(table_name.table_name());
handler->DropTable(&context, &table_name, &response);
sleep(1);
handler->CreateTable(&context, &tableschema, &response);
ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code());
fiu_enable("DropTableRequest.OnExecute.drop_table_fail", 1, NULL, 0);
handler->DropTable(&context, &table_name, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropTableRequest.OnExecute.drop_table_fail");
handler->DropTable(&context, &table_name, &response);
}
TEST_F(RpcHandlerTest, PARTITION_TEST) {
@ -436,15 +661,83 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) {
ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code());
ASSERT_EQ(partition_list.partition_array_size(), 1);
fiu_init(0);
fiu_enable("ShowPartitionsRequest.OnExecute.invalid_table_name", 1, NULL, 0);
handler->ShowPartitions(&context, &table_name, &partition_list);
fiu_disable("ShowPartitionsRequest.OnExecute.invalid_table_name");
fiu_enable("ShowPartitionsRequest.OnExecute.show_partition_fail", 1, NULL, 0);
handler->ShowPartitions(&context, &table_name, &partition_list);
fiu_disable("ShowPartitionsRequest.OnExecute.show_partition_fail");
fiu_init(0);
fiu_enable("CreatePartitionRequest.OnExecute.invalid_table_name", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.invalid_table_name");
fiu_enable("CreatePartitionRequest.OnExecute.invalid_partition_name", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.invalid_partition_name");
fiu_enable("CreatePartitionRequest.OnExecute.invalid_partition_tags", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.invalid_partition_tags");
fiu_enable("CreatePartitionRequest.OnExecute.db_already_exist", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.db_already_exist");
fiu_enable("CreatePartitionRequest.OnExecute.create_partition_fail", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.create_partition_fail");
fiu_enable("CreatePartitionRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("CreatePartitionRequest.OnExecute.throw_std_exception");
::milvus::grpc::PartitionParam partition_parm;
partition_parm.set_table_name(str_table_name);
partition_parm.set_tag(partition_tag);
fiu_enable("DropPartitionRequest.OnExecute.invalid_table_name", 1, NULL, 0);
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropPartitionRequest.OnExecute.invalid_table_name");
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code());
partition_parm.set_partition_name(partition_name);
fiu_enable("DropPartitionRequest.OnExecute.invalid_partition_tags", 1, NULL, 0);
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropPartitionRequest.OnExecute.invalid_partition_tags");
partition_parm.set_partition_name(partition_name);
fiu_enable("DropPartitionRequest.OnExecute.invalid_table_name", 1, NULL, 0);
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropPartitionRequest.OnExecute.invalid_table_name");
fiu_enable("DropPartitionRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DropPartitionRequest.OnExecute.describe_table_fail");
handler->DropPartition(&context, &partition_parm, &response);
ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code());
sleep(2);
handler->CreatePartition(&context, &partition_param, &response);
ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code());
handler->DropPartition(&context, &partition_param, &response);
ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code());
}
TEST_F(RpcHandlerTest, CMD_TEST) {
@ -461,6 +754,19 @@ TEST_F(RpcHandlerTest, CMD_TEST) {
handler->Cmd(&context, &command, &reply);
command.set_cmd("test");
handler->Cmd(&context, &command, &reply);
command.set_cmd("status");
handler->Cmd(&context, &command, &reply);
command.set_cmd("mode");
handler->Cmd(&context, &command, &reply);
command.set_cmd("build_commit_id");
handler->Cmd(&context, &command, &reply);
command.set_cmd("set_config");
handler->Cmd(&context, &command, &reply);
command.set_cmd("get_config");
handler->Cmd(&context, &command, &reply);
}
TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
@ -487,6 +793,27 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
grpc_status = handler->DeleteByDate(&context, &request, &status);
request.mutable_range()->set_end_value(CurrentTmDate(-2));
grpc_status = handler->DeleteByDate(&context, &request, &status);
fiu_init(0);
fiu_enable("DeleteByDateRequest.OnExecute.db_not_found", 1, NULL, 0);
handler->DeleteByDate(&context, &request, &status);
ASSERT_NE(status.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DeleteByDateRequest.OnExecute.db_not_found");
fiu_enable("DeleteByDateRequest.OnExecute.describe_table_fail", 1, NULL, 0);
handler->DeleteByDate(&context, &request, &status);
ASSERT_NE(status.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DeleteByDateRequest.OnExecute.describe_table_fail");
fiu_enable("DeleteByDateRequest.OnExecute.throw_std_exception", 1, NULL, 0);
handler->DeleteByDate(&context, &request, &status);
ASSERT_NE(status.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DeleteByDateRequest.OnExecute.throw_std_exception");
fiu_enable("DeleteByDateRequest.OnExecute.drop_table_fail", 1, NULL, 0);
handler->DeleteByDate(&context, &request, &status);
ASSERT_NE(status.error_code(), ::grpc::Status::OK.error_code());
fiu_disable("DeleteByDateRequest.OnExecute.drop_table_fail");
}
//////////////////////////////////////////////////////////////////////
@ -520,6 +847,27 @@ class RpcSchedulerTest : public testing::Test {
std::shared_ptr<DummyRequest> request_ptr;
};
class AsyncDummyRequest : public milvus::server::BaseRequest {
public:
milvus::Status
OnExecute() override {
return milvus::Status::OK();
}
static milvus::server::BaseRequestPtr
Create(std::string& dummy) {
return std::shared_ptr<milvus::server::BaseRequest>(new DummyRequest(dummy));
}
void TestSetStatus() {
SetStatus(milvus::SERVER_INVALID_ARGUMENT, "");
}
public:
explicit AsyncDummyRequest(std::string& dummy)
: BaseRequest(std::make_shared<milvus::server::Context>("dummy_request_id2"), dummy, true) {
}
};
} // namespace
TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
@ -527,13 +875,85 @@ TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
ASSERT_TRUE(status.ok());
milvus::server::RequestScheduler::GetInstance().Start();
// milvus::server::RequestScheduler::GetInstance().Stop();
// milvus::server::RequestScheduler::GetInstance().Start();
std::string dummy = "dql";
milvus::server::BaseRequestPtr base_task_ptr = DummyRequest::Create(dummy);
milvus::server::RequestScheduler::GetInstance().ExecRequest(base_task_ptr);
milvus::server::RequestScheduler::ExecRequest(base_task_ptr);
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(request_ptr);
fiu_init(0);
fiu_enable("RequestScheduler.ExecuteRequest.push_queue_fail", 1, NULL, 0);
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(request_ptr);
fiu_disable("RequestScheduler.ExecuteRequest.push_queue_fail");
// std::string dummy2 = "dql2";
// milvus::server::BaseRequestPtr base_task_ptr2 = DummyRequest::Create(dummy2);
// fiu_enable("RequestScheduler.PutToQueue.null_queue", 1, NULL, 0);
// milvus::server::RequestScheduler::GetInstance().ExecuteRequest(base_task_ptr2);
// fiu_disable("RequestScheduler.PutToQueue.null_queue");
std::string dummy3 = "dql3";
milvus::server::BaseRequestPtr base_task_ptr3 = DummyRequest::Create(dummy3);
fiu_enable("RequestScheduler.TakeToExecute.throw_std_exception", 1, NULL, 0);
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(base_task_ptr3);
fiu_disable("RequestScheduler.TakeToExecute.throw_std_exception");
std::string dummy4 = "dql4";
milvus::server::BaseRequestPtr base_task_ptr4 = DummyRequest::Create(dummy4);
fiu_enable("RequestScheduler.TakeToExecute.execute_fail", 1, NULL, 0);
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(base_task_ptr4);
fiu_disable("RequestScheduler.TakeToExecute.execute_fail");
std::string dummy5 = "dql5";
milvus::server::BaseRequestPtr base_task_ptr5 = DummyRequest::Create(dummy5);
fiu_enable("RequestScheduler.PutToQueue.push_null_thread", 1, NULL, 0);
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(base_task_ptr5);
fiu_disable("RequestScheduler.PutToQueue.push_null_thread");
request_ptr = nullptr;
milvus::server::RequestScheduler::GetInstance().ExecuteRequest(request_ptr);
milvus::server::BaseRequestPtr null_ptr = nullptr;
milvus::server::RequestScheduler::ExecRequest(null_ptr);
std::string async_dummy = "AsyncDummyRequest";
auto async_ptr = std::make_shared<AsyncDummyRequest>(async_dummy);
auto base_ptr = std::static_pointer_cast<milvus::server::BaseRequest>(async_ptr);
milvus::server::RequestScheduler::ExecRequest(base_ptr);
async_ptr->TestSetStatus();
milvus::server::RequestScheduler::GetInstance().Stop();
milvus::server::RequestScheduler::GetInstance().Start();
milvus::server::RequestScheduler::GetInstance().Stop();
}
TEST(RpcTest, RPC_SERVER_TEST) {
using GrpcServer = milvus::server::grpc::GrpcServer;
GrpcServer& server = GrpcServer::GetInstance();
fiu_init(0);
fiu_enable("check_config_address_fail", 1, NULL, 0);
server.Start();
sleep(2);
fiu_disable("check_config_address_fail");
server.Stop();
fiu_enable("check_config_port_fail", 1, NULL, 0);
server.Start();
sleep(2);
fiu_disable("check_config_port_fail");
server.Stop();
server.Start();
sleep(2);
server.Stop();
}
TEST(RpcTest, InterceptorHookHandlerTest) {
auto handler = std::make_shared<milvus::server::grpc::GrpcInterceptorHookHandler>();
handler->OnPostRecvInitialMetaData(nullptr, nullptr);
handler->OnPreSendMessage(nullptr, nullptr);
}

158
core/unittest/server/test_util.cpp
View File

@ -24,12 +24,17 @@
#include "utils/StringHelpFunctions.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include "utils/ThreadPool.h"
#include <gtest/gtest.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <src/utils/Exception.h>
#include <fiu-local.h>
#include <fiu-control.h>
namespace {
@ -48,6 +53,17 @@ TEST(UtilTest, EXCEPTION_TEST) {
ASSERT_EQ(ex.error_code(), milvus::SERVER_UNEXPECTED_ERROR);
std::string msg = ex.what();
ASSERT_EQ(msg, err_msg);
milvus::Exception ex1(milvus::SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex1.code(), milvus::SERVER_UNEXPECTED_ERROR);
msg = ex1.what();
ASSERT_EQ(msg, err_msg);
std::string empty_err_msg;
milvus::Exception empty_ex(milvus::SERVER_UNEXPECTED_ERROR, empty_err_msg);
ASSERT_EQ(empty_ex.code(), milvus::SERVER_UNEXPECTED_ERROR);
msg = empty_ex.what();
ASSERT_NE(msg, empty_err_msg);
}
TEST(UtilTest, SIGNAL_TEST) {
@ -64,17 +80,31 @@ TEST(UtilTest, COMMON_TEST) {
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
ASSERT_GT(thread_cnt, 0);
fiu_init(0);
fiu_enable("CommonUtil.GetSystemAvailableThreads.zero_thread", 1, NULL, 0);
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
ASSERT_GT(thread_cnt, 0);
fiu_disable("CommonUtil.GetSystemAvailableThreads.zero_thread");
std::string empty_path = "";
std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef";
milvus::Status status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
status = milvus::server::CommonUtil::CreateDirectory(empty_path);
ASSERT_TRUE(status.ok());
// test again
status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(milvus::server::CommonUtil::IsDirectoryExist(path3));
status = milvus::server::CommonUtil::DeleteDirectory(empty_path);
ASSERT_TRUE(status.ok());
status = milvus::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
// test again
@ -87,6 +117,26 @@ TEST(UtilTest, COMMON_TEST) {
std::string exe_path = milvus::server::CommonUtil::GetExePath();
ASSERT_FALSE(exe_path.empty());
fiu_enable("CommonUtil.GetExePath.readlink_fail", 1, NULL, 0);
exe_path = milvus::server::CommonUtil::GetExePath();
ASSERT_FALSE(!exe_path.empty());
fiu_disable("CommonUtil.GetExePath.readlink_fail");
fiu_enable("CommonUtil.GetExePath.exe_path_error", 1, NULL, 0);
exe_path = milvus::server::CommonUtil::GetExePath();
ASSERT_FALSE(exe_path.empty());
fiu_disable("CommonUtil.GetExePath.exe_path_error");
fiu_enable("CommonUtil.CreateDirectory.create_parent_fail", 1, NULL, 0);
status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_FALSE(status.ok());
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
fiu_enable("CommonUtil.CreateDirectory.create_dir_fail", 1, NULL, 0);
status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_FALSE(status.ok());
fiu_disable("CommonUtil.CreateDirectory.create_dir_fail");
time_t tt;
time(&tt);
tm time_struct;
@ -142,6 +192,31 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
fiu_init(0);
fiu_enable("StringHelpFunctions.SplitStringByQuote.invalid_index", 1, NULL, 0);
result.clear();
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_FALSE(status.ok());
fiu_disable("StringHelpFunctions.SplitStringByQuote.invalid_index");
fiu_enable("StringHelpFunctions.SplitStringByQuote.index_gt_last", 1, NULL, 0);
result.clear();
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
fiu_disable("StringHelpFunctions.SplitStringByQuote.index_gt_last");
fiu_enable("StringHelpFunctions.SplitStringByQuote.invalid_index2", 1, NULL, 0);
result.clear();
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_FALSE(status.ok());
fiu_disable("StringHelpFunctions.SplitStringByQuote.invalid_index2");
fiu_enable("StringHelpFunctions.SplitStringByQuote.last_is_end", 1, NULL, 0);
result.clear();
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
fiu_disable("StringHelpFunctions.SplitStringByQuote.last_is_end2");
ASSERT_TRUE(milvus::server::StringHelpFunctions::IsRegexMatch("abc", "abc"));
ASSERT_TRUE(milvus::server::StringHelpFunctions::IsRegexMatch("a8c", "a\\d."));
ASSERT_FALSE(milvus::server::StringHelpFunctions::IsRegexMatch("abc", "a\\dc"));
@ -197,11 +272,21 @@ TEST(UtilTest, TIMERECORDER_TEST) {
}
}
TEST(UtilTest, TIMERECOREDRAUTO_TEST) {
milvus::TimeRecorderAuto rc("time");
rc.RecordSection("end");
}
TEST(UtilTest, STATUS_TEST) {
auto status = milvus::Status::OK();
std::string str = status.ToString();
ASSERT_FALSE(str.empty());
status = milvus::Status(milvus::DB_SUCCESS, "success");
ASSERT_EQ(status.code(), milvus::DB_SUCCESS);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = milvus::Status(milvus::DB_ERROR, "mistake");
ASSERT_EQ(status.code(), milvus::DB_ERROR);
str = status.ToString();
@ -295,11 +380,13 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
#ifndef CUSTOMIZATION
if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
ASSERT_NE(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
continue;
}
#endif
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
}
ASSERT_EQ(
milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::MAX_VALUE + 1).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
@ -344,20 +431,36 @@ TEST(ValidationUtilTest, VALIDATE_GPU_TEST) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(100).code(), milvus::SERVER_SUCCESS);
fiu_init(0);
fiu_enable("ValidationUtil.ValidateGpuIndex.get_device_count_fail", 1, NULL, 0);
ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS);
fiu_disable("ValidationUtil.ValidateGpuIndex.get_device_count_fail");
size_t memory = 0;
ASSERT_EQ(milvus::server::ValidationUtil::GetGpuMemory(0, memory).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::GetGpuMemory(100, memory).code(), milvus::SERVER_SUCCESS);
}
#endif
TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS);
fiu_init(0);
fiu_enable("ValidationUtil.ValidateIpAddress.error_ip_result", 1, NULL, 0);
ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS);
fiu_disable("ValidationUtil.ValidateIpAddress.error_ip_result");
}
TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsNumber("1234").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("not number").code(), milvus::SERVER_SUCCESS);
fiu_init(0);
fiu_enable("ValidationUtil.ValidateStringIsNumber.throw_exception", 1, NULL, 0);
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("122").code(), milvus::SERVER_SUCCESS);
fiu_disable("ValidationUtil.ValidateStringIsNumber.throw_exception");
}
TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) {
@ -386,13 +489,28 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) {
std::string dir1 = "/tmp/milvus_test";
std::string dir2 = "/tmp/milvus_test/log_test";
std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log",
"log_trace.log", "log_error.log", "log_fatal.log"};
"log_trace.log", "log_error.log", "log_fatal.log"};
el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning,
el::Level::Trace, el::Level::Error, el::Level::Fatal};
el::Level::Trace, el::Level::Error, el::Level::Fatal};
mkdir(dir1.c_str(), S_IRWXU);
mkdir(dir2.c_str(), S_IRWXU);
// [&]() {
//// std::string tmp = dir2 + "/" + filename[0]+"*@%$";
// std::string tmp = dir2 + "/" + filename[0] + "*$";
// std::ofstream file;
// file.open(tmp.c_str());
// file << "test" << std::endl;
// milvus::server::RolloutHandler(tmp.c_str(), 0, el::Level::Unknown);
// tmp.append(".1");
// std::ifstream file2;
// file2.open(tmp);
// std::string tmp2;
// file2 >> tmp2;
// ASSERT_EQ(tmp2, "test");
// }();
for (int i = 0; i < 6; ++i) {
std::string tmp = dir2 + "/" + filename[i];
@ -410,5 +528,41 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) {
file2 >> tmp2;
ASSERT_EQ(tmp2, "test");
}
[&]() {
std::string tmp = dir2 + "/" + filename[0];
std::ofstream file;
file.open(tmp.c_str());
file << "test" << std::endl;
milvus::server::RolloutHandler(tmp.c_str(), 0, el::Level::Unknown);
tmp.append(".1");
std::ifstream file2;
file2.open(tmp);
std::string tmp2;
file2 >> tmp2;
ASSERT_EQ(tmp2, "test");
}();
boost::filesystem::remove_all(dir2);
}
TEST(UtilTest, THREADPOOL_TEST) {
auto thread_pool_ptr = std::make_unique<milvus::ThreadPool>(3);
auto fun = [](int i) {
sleep(1);
};
for (int i = 0; i < 10; ++i) {
thread_pool_ptr->enqueue(fun, i);
}
fiu_init(0);
fiu_enable("ThreadPool.enqueue.stop_is_true", 1, NULL, 0);
try {
thread_pool_ptr->enqueue(fun, -1);
} catch (std::exception& err) {
std::cout << "catch an error here" << std::endl;
}
fiu_disable("ThreadPool.enqueue.stop_is_true");
thread_pool_ptr.reset();
}

8
core/unittest/server/utils.cpp
View File

@ -58,7 +58,7 @@ static const char* VALID_CONFIG_STR =
"engine_config:\n"
" use_blas_threshold: 20 \n"
"\n"
#ifdef MILVUS_GPU_VERSION
#ifdef MILVUS_GPU_VERSION
"gpu_resource_config:\n"
" enable: true # whether to enable GPU resources\n"
" cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n"
@ -66,7 +66,11 @@ static const char* VALID_CONFIG_STR =
" - gpu0\n"
" build_index_resources: # define the GPU devices used for index building, must be in format gpux\n"
" - gpu0\n"
#endif
#endif
"\n"
"sequence_config:\n"
" - seq1\n"
" - seq2\n"
"\n";
static const char* INVALID_CONFIG_STR = "*INVALID*";

73
core/unittest/wrapper/test_hybrid_index.cpp
View File

@ -21,6 +21,8 @@
#include "wrapper/utils.h"
#include <gtest/gtest.h>
#include <fiu-local.h>
#include <fiu-control.h>
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
using ::testing::Combine;
@ -131,4 +133,75 @@ TEST_F(KnowhereHybrid, test_interface) {
}
}
TEST_F(KnowhereHybrid, test_invalid_index) {
assert(!xb.empty());
fiu_init(0);
index_type = milvus::engine::IndexType::FAISS_IVFSQ8_HYBRID;
fiu_enable("GetVecIndexFactory.IVFSQHybrid.mock", 1, nullptr, 0);
index_ = GetVecIndexFactory(index_type);
fiu_disable("GetVecIndexFactory.IVFSQHybrid.mock");
conf = ParamGenerator::GetInstance().Gen(index_type);
auto elems = nq * k;
std::vector<int64_t> res_ids(elems);
std::vector<float> res_dis(elems);
conf->gpu_id = DEVICEID;
conf->d = dim;
conf->k = k;
auto status = index_->BuildAll(nb, xb.data(), ids.data(), conf);
ASSERT_FALSE(status.ok());
auto quantizer = std::make_shared<knowhere::Quantizer>();
auto quantizer_cfg = std::make_shared<knowhere::QuantizerCfg>();
index_->LoadQuantizer(quantizer_cfg);
status = index_->SetQuantizer(quantizer);
ASSERT_FALSE(status.ok());
fiu_enable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception", 1, nullptr, 0);
status = index_->SetQuantizer(quantizer);
ASSERT_FALSE(status.ok());
fiu_disable("IVFHybridIndex.SetQuantizer.throw_knowhere_exception");
fiu_enable("IVFHybridIndex.SetQuantizer.throw_std_exception", 1, nullptr, 0);
status = index_->SetQuantizer(quantizer);
ASSERT_FALSE(status.ok());
fiu_disable("IVFHybridIndex.SetQuantizer.throw_std_exception");
status = index_->UnsetQuantizer();
ASSERT_FALSE(status.ok());
fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception", 1, nullptr, 0);
status = index_->UnsetQuantizer();
ASSERT_FALSE(status.ok());
fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_knowhere_exception");
fiu_enable("IVFHybridIndex.UnsetQuantizer.throw_std_exception", 1, nullptr, 0);
status = index_->UnsetQuantizer();
ASSERT_FALSE(status.ok());
fiu_disable("IVFHybridIndex.UnsetQuantizer.throw_std_exception");
auto vecindex = index_->LoadData(quantizer, quantizer_cfg);
ASSERT_EQ(vecindex, nullptr);
fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0);
vecindex = index_->LoadData(quantizer, quantizer_cfg);
ASSERT_EQ(vecindex, nullptr);
fiu_disable("IVFHybridIndex.LoadData.throw_std_exception");
fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0);
vecindex = index_->LoadData(quantizer, quantizer_cfg);
ASSERT_EQ(vecindex, nullptr);
fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception");
index_->CopyToGpuWithQuantizer(DEVICEID, conf);
fiu_enable("IVFHybridIndex.LoadData.throw_knowhere_exception", 1, nullptr, 0);
index_->CopyToGpuWithQuantizer(DEVICEID, conf);
fiu_disable("IVFHybridIndex.LoadData.throw_knowhere_exception");
fiu_enable("IVFHybridIndex.LoadData.throw_std_exception", 1, nullptr, 0);
index_->CopyToGpuWithQuantizer(DEVICEID, conf);
fiu_disable("IVFHybridIndex.LoadData.throw_std_exception");
}
#endif

25
core/unittest/wrapper/test_knowhere.cpp
View File

@ -19,6 +19,8 @@
#include "wrapper/KnowhereResource.h"
#include "wrapper/utils.h"
#include <fiu-local.h>
#include <fiu-control.h>
#include <gtest/gtest.h>
TEST_F(KnowhereTest, KNOWHERE_RESOURCE_TEST) {
@ -30,4 +32,27 @@ TEST_F(KnowhereTest, KNOWHERE_RESOURCE_TEST) {
milvus::engine::KnowhereResource::Initialize();
milvus::engine::KnowhereResource::Finalize();
#ifdef MILVUS_GPU_VERSION
fiu_init(0);
fiu_enable("check_config_gpu_resource_enable_fail", 1, NULL, 0);
s = milvus::engine::KnowhereResource::Initialize();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("KnowhereResource.Initialize.disable_gpu", 1, NULL, 0);
s = milvus::engine::KnowhereResource::Initialize();
ASSERT_TRUE(s.ok());
fiu_disable("KnowhereResource.Initialize.disable_gpu");
fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0);
s = milvus::engine::KnowhereResource::Initialize();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_build_index_fail");
fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0);
s = milvus::engine::KnowhereResource::Initialize();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_search_fail");
#endif
}

157
core/unittest/wrapper/test_wrapper.cpp
View File

@ -18,13 +18,18 @@
#include "easyloggingpp/easylogging++.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "wrapper/WrapperException.h"
#endif
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include <fiu-control.h>
#include <fiu-local.h>
#include <gtest/gtest.h>
INITIALIZE_EASYLOGGINGPP
@ -78,18 +83,29 @@ INSTANTIATE_TEST_CASE_P(
Values(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
#ifdef MILVUS_GPU_VERSION
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10),
// std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10),
// std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
#endif
// std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10),
// std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10)));
#ifdef MILVUS_GPU_VERSION
TEST_P(KnowhereWrapperTest, WRAPPER_EXCEPTION_TEST) {
std::string err_msg = "failed";
milvus::engine::WrapperException ex(err_msg);
std::string msg = ex.what();
EXPECT_EQ(msg, err_msg);
}
#endif
// std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10),
// std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10)));
TEST_P(KnowhereWrapperTest, BASE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
@ -102,6 +118,43 @@ TEST_P(KnowhereWrapperTest, BASE_TEST) {
index_->BuildAll(nb, xb.data(), ids.data(), conf);
index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), searchconf);
AssertResult(res_ids, res_dis);
{
index_->GetDeviceId();
fiu_init(0);
fiu_enable("VecIndexImpl.BuildAll.throw_knowhere_exception", 1, NULL, 0);
fiu_enable("BFIndex.BuildAll.throw_knowhere_exception", 1, NULL, 0);
fiu_enable("IVFMixIndex.BuildAll.throw_knowhere_exception", 1, NULL, 0);
auto s = index_->BuildAll(nb, xb.data(), ids.data(), conf);
fiu_disable("IVFMixIndex.BuildAll.throw_knowhere_exception");
fiu_disable("BFIndex.BuildAll.throw_knowhere_exception");
fiu_disable("VecIndexImpl.BuildAll.throw_knowhere_exception");
fiu_enable("VecIndexImpl.BuildAll.throw_std_exception", 1, NULL, 0);
fiu_enable("BFIndex.BuildAll.throw_std_exception", 1, NULL, 0);
fiu_enable("IVFMixIndex.BuildAll.throw_std_exception", 1, NULL, 0);
s = index_->BuildAll(nb, xb.data(), ids.data(), conf);
fiu_disable("IVFMixIndex.BuildAll.throw_std_exception");
fiu_disable("BFIndex.BuildAll.throw_std_exception");
fiu_disable("VecIndexImpl.BuildAll.throw_std_exception");
fiu_enable("VecIndexImpl.Add.throw_knowhere_exception", 1, NULL, 0);
s = index_->Add(nb, xb.data(), ids.data());
fiu_disable("VecIndexImpl.Add.throw_knowhere_exception");
fiu_enable("VecIndexImpl.Add.throw_std_exception", 1, NULL, 0);
s = index_->Add(nb, xb.data(), ids.data());
fiu_disable("VecIndexImpl.Add.throw_std_exception");
fiu_enable("VecIndexImpl.Search.throw_knowhere_exception", 1, NULL, 0);
s = index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), searchconf);
fiu_disable("VecIndexImpl.Search.throw_knowhere_exception");
fiu_enable("VecIndexImpl.Search.throw_std_exception", 1, NULL, 0);
s = index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), searchconf);
fiu_disable("VecIndexImpl.Search.throw_std_exception");
}
}
#ifdef MILVUS_GPU_VERSION
@ -136,9 +189,11 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
AssertResult(res_ids, res_dis);
}
}
#endif
TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
std::cout << "type: " << static_cast<int>(index_type) << std::endl;
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
@ -175,6 +230,25 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), searchconf);
AssertResult(res_ids, res_dis);
}
{
std::string file_location = "/tmp/knowhere_gpu_file";
fiu_init(0);
fiu_enable("VecIndex.write_index.throw_knowhere_exception", 1, NULL, 0);
auto s = write_index(index_, file_location);
ASSERT_FALSE(s.ok());
fiu_disable("VecIndex.write_index.throw_knowhere_exception");
fiu_enable("VecIndex.write_index.throw_std_exception", 1, NULL, 0);
s = write_index(index_, file_location);
ASSERT_FALSE(s.ok());
fiu_disable("VecIndex.write_index.throw_std_exception");
fiu_enable("VecIndex.write_index.throw_no_space_exception", 1, NULL, 0);
s = write_index(index_, file_location);
ASSERT_FALSE(s.ok());
fiu_disable("VecIndex.write_index.throw_no_space_exception");
}
}
#include "wrapper/ConfAdapter.h"
@ -198,6 +272,67 @@ TEST(whatever, test_config) {
auto bkt_conf = std::make_shared<milvus::engine::SPTAGBKTConfAdapter>();
bkt_conf->Match(conf);
bkt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_BKT_RNT_CPU);
auto config_mgr = milvus::engine::AdapterMgr::GetInstance();
try {
config_mgr.GetAdapter(milvus::engine::IndexType::INVALID);
} catch (std::exception& e) {
std::cout << "catch an expected exception" << std::endl;
}
conf.size = 1000000.0;
conf.nlist = 10;
auto ivf_conf = std::make_shared<milvus::engine::IVFConfAdapter>();
ivf_conf->Match(conf);
conf.nprobe = -1;
ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFFLAT_GPU);
conf.nprobe = 4096;
ivf_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU);
auto ivf_pq_conf = std::make_shared<milvus::engine::IVFPQConfAdapter>();
conf.metric_type = knowhere::METRICTYPE::IP;
try {
ivf_pq_conf->Match(conf);
} catch (std::exception& e) {
std::cout << "catch an expected exception" << std::endl;
}
conf.metric_type = knowhere::METRICTYPE::L2;
fiu_init(0);
fiu_enable("IVFPQConfAdapter.Match.empty_resset", 1, NULL, 0);
try {
ivf_pq_conf->Match(conf);
} catch (std::exception& e) {
std::cout << "catch an expected exception" << std::endl;
}
fiu_disable("IVFPQConfAdapter.Match.empty_resset");
conf.nprobe = -1;
try {
ivf_pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU);
} catch (std::exception& e) {
std::cout << "catch an expected exception" << std::endl;
}
}
#include "wrapper/VecImpl.h"
TEST(BFIndex, test_bf_index_fail) {
auto bf_ptr = std::make_shared<milvus::engine::BFIndex>(nullptr);
auto float_vec = bf_ptr->GetRawVectors();
ASSERT_EQ(float_vec, nullptr);
milvus::engine::Config config;
fiu_init(0);
fiu_enable("BFIndex.Build.throw_knowhere_exception", 1, NULL, 0);
auto err_code = bf_ptr->Build(config);
ASSERT_EQ(err_code, milvus::KNOWHERE_UNEXPECTED_ERROR);
fiu_disable("BFIndex.Build.throw_knowhere_exception");
fiu_enable("BFIndex.Build.throw_std_exception", 1, NULL, 0);
err_code = bf_ptr->Build(config);
ASSERT_EQ(err_code, milvus::KNOWHERE_ERROR);
fiu_disable("BFIndex.Build.throw_std_exception");
}
// #include "knowhere/index/vector_index/IndexIDMAP.h"