admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into 'branch-0.4.0'

Browse Source 
refine unittest code

See merge request megasearch/milvus!547

Former-commit-id: e64a348ccb54a880776cb91a757793a84adbc69b
This commit is contained in:
peng.xu 2019-09-11 11:54:59 +08:00
commit 26e63ae796
28 changed files with 130 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/conf/server_config.template
View File

@ -1,7 +1,6 @@
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
@ -18,6 +17,7 @@ db_config:
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
metric_config:
is_startup: off # if monitoring start: on, off
@ -33,8 +33,6 @@ cache_config:
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
gpu_ids: # gpu id
- 0
engine_config:
use_blas_threshold: 20

39
cpp/src/cache/GpuCacheMgr.cpp vendored
View File

@ -17,36 +17,14 @@ std::mutex GpuCacheMgr::mutex_;
std::unordered_map<uint64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
namespace {
constexpr int64_t unit = 1024 * 1024 * 1024;
std::vector<uint64_t> load() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
auto conf_gpu_ids = config.GetSequence(server::CONFIG_GPU_IDS);
std::vector<uint64_t > gpu_ids;
for (auto gpu_id : conf_gpu_ids) {
gpu_ids.push_back(std::atoi(gpu_id.c_str()));
}
return gpu_ids;
}
}
bool GpuCacheMgr::GpuIdInConfig(uint64_t gpu_id) {
static std::vector<uint64_t > ids = load();
for (auto id : ids) {
if (gpu_id == id) return true;
}
return false;
constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
}
GpuCacheMgr::GpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 2);
cap *= unit;
cap *= G_BYTE;
cache_ = std::make_shared<Cache>(cap, 1UL<<32);
double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85);
@ -58,6 +36,19 @@ GpuCacheMgr::GpuCacheMgr() {
}
}
CacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
instance_.insert(std::pair<uint64_t, GpuCacheMgrPtr>(gpu_id, std::make_shared<GpuCacheMgr>()));
}
return instance_[gpu_id].get();
} else {
std::lock_guard<std::mutex> lock(mutex_);
return instance_[gpu_id].get();
}
}
void GpuCacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
//TODO: copy data to gpu
if (cache_ == nullptr) {

16
cpp/src/cache/GpuCacheMgr.h vendored
View File

@ -19,21 +19,7 @@ class GpuCacheMgr : public CacheMgr {
public:
GpuCacheMgr();
static bool GpuIdInConfig(uint64_t gpu_id);
static CacheMgr* GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
if (GpuIdInConfig(gpu_id)) {
instance_.insert(std::pair<uint64_t, GpuCacheMgrPtr>(gpu_id, std::make_shared<GpuCacheMgr>()));
} else {
return nullptr;
}
}
}
return instance_[gpu_id].get();
}
static CacheMgr* GetInstance(uint64_t gpu_id);
void InsertItem(const std::string& key, const DataObjPtr& data) override;

23
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -168,22 +168,13 @@ void PrometheusMetrics::CPUTemperature() {
}
void PrometheusMetrics::GpuCacheUsageGaugeSet() {
if(!startup_) return;
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
auto conf_gpu_ids = config.GetSequence(server::CONFIG_GPU_IDS);
std::vector<uint64_t > gpu_ids;
for (auto gpu_id : conf_gpu_ids) {
gpu_ids.push_back(std::atoi(gpu_id.c_str()));
}
for(auto i = 0; i < gpu_ids.size(); ++i) {
uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
gpu_cache.Set(cache_usage * 100 / cache_capacity);
}
// std::vector<uint64_t > gpu_ids = {0};
// for(auto i = 0; i < gpu_ids.size(); ++i) {
// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
// gpu_cache.Set(cache_usage * 100 / cache_capacity);
// }
}
}

41
cpp/src/server/ServerConfig.cpp
View File

@ -113,18 +113,6 @@ ServerConfig::CheckServerConfig() {
}
}
std::string gpu_index_str = server_config.GetValue(CONFIG_GPU_INDEX, "0");
if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl;
okay = false;
} else {
int32_t gpu_index = std::stol(gpu_index_str);
if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl;
okay = false;
}
}
std::string mode = server_config.GetValue(CONFIG_CLUSTER_MODE, "single");
if (mode != "single" && mode != "cluster" && mode != "read_only") {
std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl;
@ -214,6 +202,18 @@ ServerConfig::CheckDBConfig() {
}
}
std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, "0");
if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl;
okay = false;
} else {
int32_t gpu_index = std::stol(gpu_index_str);
if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl;
okay = false;
}
}
return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
}
@ -313,7 +313,7 @@ ServerConfig::CheckCacheConfig() {
else {
uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str);
gpu_cache_capacity *= GB;
int gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
size_t gpu_memory;
if (ValidationUtil::GetGpuMemory(gpu_index, gpu_memory) != SERVER_SUCCESS) {
std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl;
@ -340,19 +340,6 @@ ServerConfig::CheckCacheConfig() {
okay = false;
}
auto conf_gpu_ids = cache_config.GetSequence(server::CONFIG_GPU_IDS);
for (std::string &gpu_id : conf_gpu_ids) {
if (ValidationUtil::ValidateStringIsNumber(gpu_id) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_id " << gpu_id << " is not a number" << std::endl;
okay = false;
}
else if (ValidationUtil::ValidateGpuIndex(std::stol(gpu_id)) != SERVER_SUCCESS) {
std::cerr << "ERROR: gpu_id " << gpu_id << " is invalid" << std::endl;
okay = false;
}
}
return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
}
@ -483,7 +470,7 @@ ServerConfig::CheckResourceConfig() {
}
}
else if (type == "GPU") {
int build_index_gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
int build_index_gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
if (device_id == build_index_gpu_index) {
resource_valid_flag = true;
}

3
cpp/src/server/ServerConfig.h
View File

@ -18,7 +18,6 @@ static const char* CONFIG_SERVER = "server_config";
static const char* CONFIG_SERVER_ADDRESS = "address";
static const char* CONFIG_SERVER_PORT = "port";
static const char* CONFIG_CLUSTER_MODE = "mode";
static const char* CONFIG_GPU_INDEX = "gpu_index";
static const char* CONFIG_TIME_ZONE = "time_zone";
static const char* CONFIG_DB = "db_config";
@ -29,6 +28,7 @@ static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_LOG = "log_config";
@ -37,7 +37,6 @@ static const char* CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity";
static const char* CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity";
static const char* CACHE_FREE_PERCENT = "cpu_cache_free_percent";
static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately";
static const char* CONFIG_GPU_IDS = "gpu_ids";
static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent";
static const char* CONFIG_METRIC = "metric_config";

4
cpp/unittest/db/db_tests.cpp
View File

@ -25,8 +25,8 @@ namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;

3
cpp/unittest/db/engine_test.cpp
View File

@ -105,9 +105,6 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Dimension(), dimension);
ASSERT_EQ(engine_ptr->Count(), ids.size());
// server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
// config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
//
// status = engine_ptr->CopyToGpu(0);
// //ASSERT_TRUE(status.ok());
//

6
cpp/unittest/db/mem_test.cpp
View File

@ -24,8 +24,6 @@ namespace {
static std::string TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
std::string GenTableName() {
auto now = std::chrono::system_clock::now();
@ -212,7 +210,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::map<int64_t, std::vector<float>> search_vectors;
{
engine::IDNumbers vector_ids;
int64_t nb = 1024000;
int64_t nb = 100000;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
@ -224,7 +222,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::mt19937 gen(rd());
std::uniform_int_distribution<int64_t> dis(0, nb - 1);
int64_t num_query = 20;
int64_t num_query = 10;
for (int64_t i = 0; i < num_query; ++i) {
int64_t index = dis(gen);
std::vector<float> search;

4
cpp/unittest/db/mysql_db_test.cpp
View File

@ -22,8 +22,8 @@ namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 10000;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;

6
cpp/unittest/db/search_test.cpp
View File

@ -288,7 +288,7 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
DoTopk(5, 10, 4, false);
DoTopk(20005, 998, 123, true);
DoTopk(9987, 12, 10, false);
DoTopk(77777, 1000, 1, false);
DoTopk(5432, 8899, 8899, true);
// DoTopk(9987, 12, 10, false);
// DoTopk(77777, 1000, 1, false);
// DoTopk(5432, 8899, 8899, true);
}

3
cpp/unittest/db/utils.cpp
View File

@ -66,9 +66,6 @@ engine::Options BaseTest::GetOptions() {
void DBTest::SetUp() {
BaseTest::SetUp();
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
auto res_mgr = engine::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false));

12
cpp/unittest/knowhere/knowhere_test.cpp
View File

@ -142,7 +142,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
)
);
TEST_P(KnowhereWrapperTest, base_test) {
TEST_P(KnowhereWrapperTest, BASE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
@ -154,7 +154,7 @@ TEST_P(KnowhereWrapperTest, base_test) {
AssertResult(res_ids, res_dis);
}
TEST_P(KnowhereWrapperTest, to_gpu_test) {
TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
@ -174,7 +174,7 @@ TEST_P(KnowhereWrapperTest, to_gpu_test) {
}
{
std::string file_location = "/tmp/test_gpu_file";
std::string file_location = "/tmp/knowhere_gpu_file";
write_index(index_, file_location);
auto new_index = read_index(file_location);
@ -186,11 +186,11 @@ TEST_P(KnowhereWrapperTest, to_gpu_test) {
}
}
TEST_P(KnowhereWrapperTest, to_cpu_test) {
TEST_P(KnowhereWrapperTest, TO_CPU_TEST) {
// dev
}
TEST_P(KnowhereWrapperTest, serialize) {
TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
auto elems = nq * k;
@ -215,7 +215,7 @@ TEST_P(KnowhereWrapperTest, serialize) {
}
{
std::string file_location = "/tmp/whatever";
std::string file_location = "/tmp/knowhere";
write_index(index_, file_location);
auto new_index = read_index(file_location);
EXPECT_EQ(new_index->GetType(), ConvertToCpuIndexType(index_type));

4
cpp/unittest/metrics/metrics_test.cpp
View File

@ -24,7 +24,7 @@
using namespace zilliz::milvus;
TEST_F(MetricTest, Metric_Tes) {
TEST_F(MetricTest, METRIC_TEST) {
server::ConfigNode &configNode = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_METRIC);
configNode.SetValue(server::CONFIG_METRIC_COLLECTOR, "zabbix");
server::Metrics::GetInstance();
@ -122,7 +122,7 @@ TEST_F(MetricTest, Metric_Tes) {
delete [] qxb;
};
TEST_F(MetricTest, Collector_Metrics_Test){
TEST_F(MetricTest, COLLECTOR_METRICS_TEST){
engine::Status status = engine::Status::OK();
server::CollectInsertMetrics insert_metrics0(0, status);
status = engine::Status(DB_ERROR, "error");

2
cpp/unittest/scheduler/algorithm_test.cpp
View File

@ -53,7 +53,7 @@ class AlgorithmTest : public testing::Test {
ResourceMgrPtr res_mgr_;
};
TEST_F(AlgorithmTest, ShortestPath_test) {
TEST_F(AlgorithmTest, SHORTESTPATH_TEST) {
std::vector<std::string> sp;
uint64_t cost;
cost = ShortestPath(disk_.lock(), gpu_0_.lock(), res_mgr_, sp);

8
cpp/unittest/scheduler/event_test.cpp
View File

@ -15,7 +15,7 @@ namespace zilliz {
namespace milvus {
namespace engine {
TEST(EventTest, start_up_event) {
TEST(EventTest, START_UP_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<StartUpEvent>(res);
ASSERT_FALSE(event->Dump().empty());
@ -23,7 +23,7 @@ TEST(EventTest, start_up_event) {
std::cout << *EventPtr(event);
}
TEST(EventTest, load_completed_event) {
TEST(EventTest, LOAD_COMPLETED_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<LoadCompletedEvent>(res, nullptr);
ASSERT_FALSE(event->Dump().empty());
@ -31,7 +31,7 @@ TEST(EventTest, load_completed_event) {
std::cout << *EventPtr(event);
}
TEST(EventTest, finish_task_event) {
TEST(EventTest, FINISH_TASK_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<FinishTaskEvent>(res, nullptr);
ASSERT_FALSE(event->Dump().empty());
@ -40,7 +40,7 @@ TEST(EventTest, finish_task_event) {
}
TEST(EventTest, tasktable_updated_event) {
TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
ResourceWPtr res(ResourcePtr(nullptr));
auto event = std::make_shared<TaskTableUpdatedEvent>(res);
ASSERT_FALSE(event->Dump().empty());

8
cpp/unittest/scheduler/node_test.cpp
View File

@ -28,7 +28,7 @@ protected:
NodePtr isolated_node2_;
};
TEST_F(NodeTest, add_neighbour) {
TEST_F(NodeTest, ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
@ -37,7 +37,7 @@ TEST_F(NodeTest, add_neighbour) {
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
}
TEST_F(NodeTest, repeat_add_neighbour) {
TEST_F(NodeTest, REPEAT_ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
@ -47,7 +47,7 @@ TEST_F(NodeTest, repeat_add_neighbour) {
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
}
TEST_F(NodeTest, get_neighbours) {
TEST_F(NodeTest, GET_NEIGHBOURS) {
{
bool n2 = false, n3 = false;
auto node1_neighbours = node1_->GetNeighbours();
@ -72,7 +72,7 @@ TEST_F(NodeTest, get_neighbours) {
}
}
TEST_F(NodeTest, dump) {
TEST_F(NodeTest, DUMP) {
std::cout << node1_->Dump();
ASSERT_FALSE(node1_->Dump().empty());

2
cpp/unittest/scheduler/normal_test.cpp
View File

@ -11,7 +11,7 @@
using namespace zilliz::milvus::engine;
TEST(normal_test, inst_test) {
TEST(NormalTest, INST_TEST) {
// ResourceMgr only compose resources, provide unified event
auto res_mgr = ResMgrInst::GetInstance();

2
cpp/unittest/scheduler/resource_factory_test.cpp
View File

@ -4,7 +4,7 @@
using namespace zilliz::milvus::engine;
TEST(resource_factory_test, create) {
TEST(ResourceFactoryTest, CREATE) {
auto disk = ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ResourceFactory::Create("gpu", "GPU", 0);

32
cpp/unittest/scheduler/resource_mgr_test.cpp
View File

@ -44,19 +44,19 @@ protected:
ResourcePtr gpu_res;
};
TEST_F(ResourceMgrBaseTest, add) {
TEST_F(ResourceMgrBaseTest, ADD) {
auto resource = std::make_shared<TestResource>("test", 0, true, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
}
TEST_F(ResourceMgrBaseTest, add_disk) {
TEST_F(ResourceMgrBaseTest, ADD_DISK) {
auto resource = std::make_shared<DiskResource>("disk", 0, true, true);
auto ret = empty_mgr_->Add(ResourcePtr(resource));
ASSERT_EQ(ret.lock(), resource);
}
TEST_F(ResourceMgrBaseTest, connect) {
TEST_F(ResourceMgrBaseTest, CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
empty_mgr_->Add(resource1);
@ -66,7 +66,7 @@ TEST_F(ResourceMgrBaseTest, connect) {
}
TEST_F(ResourceMgrBaseTest, invalid_connect) {
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
empty_mgr_->Add(resource1);
@ -76,19 +76,19 @@ TEST_F(ResourceMgrBaseTest, invalid_connect) {
}
TEST_F(ResourceMgrBaseTest, clear) {
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
ASSERT_EQ(mgr1_->GetNumOfResource(), 0);
}
TEST_F(ResourceMgrBaseTest, get_disk_resources) {
TEST_F(ResourceMgrBaseTest, GET_DISK_RESOURCES) {
auto disks = mgr1_->GetDiskResources();
ASSERT_EQ(disks.size(), 1);
ASSERT_EQ(disks[0].lock(), disk_res);
}
TEST_F(ResourceMgrBaseTest, get_all_resources) {
TEST_F(ResourceMgrBaseTest, GET_ALL_RESOURCES) {
bool disk = false, cpu = false, gpu = false;
auto resources = mgr1_->GetAllResources();
ASSERT_EQ(resources.size(), 3);
@ -103,13 +103,13 @@ TEST_F(ResourceMgrBaseTest, get_all_resources) {
ASSERT_TRUE(gpu);
}
TEST_F(ResourceMgrBaseTest, get_compute_resources) {
TEST_F(ResourceMgrBaseTest, GET_COMPUTE_RESOURCES) {
auto compute_resources = mgr1_->GetComputeResources();
ASSERT_EQ(compute_resources.size(), 1);
ASSERT_EQ(compute_resources[0], gpu_res);
}
TEST_F(ResourceMgrBaseTest, get_resource_by_type_and_deviceid) {
TEST_F(ResourceMgrBaseTest, GET_RESOURCE_BY_TYPE_AND_DEVICEID) {
auto cpu = mgr1_->GetResource(ResourceType::CPU, 1);
ASSERT_EQ(cpu, cpu_res);
@ -117,7 +117,7 @@ TEST_F(ResourceMgrBaseTest, get_resource_by_type_and_deviceid) {
ASSERT_EQ(invalid, nullptr);
}
TEST_F(ResourceMgrBaseTest, get_resource_by_name) {
TEST_F(ResourceMgrBaseTest, GET_RESOURCE_BY_NAME) {
auto disk = mgr1_->GetResource("disk");
ASSERT_EQ(disk, disk_res);
@ -125,26 +125,26 @@ TEST_F(ResourceMgrBaseTest, get_resource_by_name) {
ASSERT_EQ(invalid, nullptr);
}
TEST_F(ResourceMgrBaseTest, get_num_of_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumOfResource(), 0);
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
}
TEST_F(ResourceMgrBaseTest, get_num_of_compute_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_COMPUTE_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumOfComputeResource(), 0);
ASSERT_EQ(mgr1_->GetNumOfComputeResource(), 1);
}
TEST_F(ResourceMgrBaseTest, get_num_of_gpu_resource) {
TEST_F(ResourceMgrBaseTest, GET_NUM_OF_GPU_RESOURCE) {
ASSERT_EQ(empty_mgr_->GetNumGpuResource(), 0);
ASSERT_EQ(mgr1_->GetNumGpuResource(), 1);
}
TEST_F(ResourceMgrBaseTest, dump) {
TEST_F(ResourceMgrBaseTest, DUMP) {
ASSERT_FALSE(mgr1_->Dump().empty());
}
TEST_F(ResourceMgrBaseTest, dump_tasktables) {
TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
ASSERT_FALSE(mgr1_->DumpTaskTables().empty());
}
@ -169,7 +169,7 @@ class ResourceMgrAdvanceTest : public testing::Test {
ResourcePtr disk_res;
};
TEST_F(ResourceMgrAdvanceTest, register_subscriber) {
TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
bool flag = false;
auto callback = [&](EventPtr event) {
flag = true;

20
cpp/unittest/scheduler/resource_test.cpp
View File

@ -46,42 +46,42 @@ protected:
ResourcePtr both_disable_ = nullptr;
};
TEST_F(ResourceBaseTest, name) {
TEST_F(ResourceBaseTest, NAME) {
ASSERT_EQ(only_loader_->name(), name1);
ASSERT_EQ(only_executor_->name(), name2);
ASSERT_EQ(both_enable_->name(), name3);
ASSERT_EQ(both_disable_->name(), name4);
}
TEST_F(ResourceBaseTest, type) {
TEST_F(ResourceBaseTest, TYPE) {
ASSERT_EQ(only_loader_->type(), ResourceType::DISK);
ASSERT_EQ(only_executor_->type(), ResourceType::CPU);
ASSERT_EQ(both_enable_->type(), ResourceType::GPU);
ASSERT_EQ(both_disable_->type(), ResourceType::TEST);
}
TEST_F(ResourceBaseTest, device_id) {
TEST_F(ResourceBaseTest, DEVICE_ID) {
ASSERT_EQ(only_loader_->device_id(), id1);
ASSERT_EQ(only_executor_->device_id(), id2);
ASSERT_EQ(both_enable_->device_id(), id3);
ASSERT_EQ(both_disable_->device_id(), id4);
}
TEST_F(ResourceBaseTest, has_loader) {
TEST_F(ResourceBaseTest, HAS_LOADER) {
ASSERT_TRUE(only_loader_->HasLoader());
ASSERT_FALSE(only_executor_->HasLoader());
ASSERT_TRUE(both_enable_->HasLoader());
ASSERT_FALSE(both_disable_->HasLoader());
}
TEST_F(ResourceBaseTest, has_executor) {
TEST_F(ResourceBaseTest, HAS_EXECUTOR) {
ASSERT_FALSE(only_loader_->HasExecutor());
ASSERT_TRUE(only_executor_->HasExecutor());
ASSERT_TRUE(both_enable_->HasExecutor());
ASSERT_FALSE(both_disable_->HasExecutor());
}
TEST_F(ResourceBaseTest, dump) {
TEST_F(ResourceBaseTest, DUMP) {
ASSERT_FALSE(only_loader_->Dump().empty());
ASSERT_FALSE(only_executor_->Dump().empty());
ASSERT_FALSE(both_enable_->Dump().empty());
@ -165,7 +165,7 @@ protected:
std::condition_variable cv_;
};
TEST_F(ResourceAdvanceTest, disk_resource_test) {
TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
@ -190,7 +190,7 @@ TEST_F(ResourceAdvanceTest, disk_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, cpu_resource_test) {
TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
@ -215,7 +215,7 @@ TEST_F(ResourceAdvanceTest, cpu_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, gpu_resource_test) {
TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
@ -240,7 +240,7 @@ TEST_F(ResourceAdvanceTest, gpu_resource_test) {
}
}
TEST_F(ResourceAdvanceTest, test_resource_test) {
TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
const uint64_t NUM = 100;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;

2
cpp/unittest/scheduler/schedinst_test.cpp
View File

@ -68,7 +68,7 @@ protected:
const std::string CONFIG_FILE = "/tmp/milvus_sched_test/config.yaml";
};
TEST_F(SchedInstTest, simple_gpu) {
TEST_F(SchedInstTest, SIMPLE_GPU) {
StartSchedulerService();
}

10
cpp/unittest/scheduler/scheduler_test.cpp
View File

@ -94,10 +94,6 @@ class SchedulerTest : public testing::Test {
protected:
void
SetUp() override {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
config.AddSequenceItem(server::CONFIG_GPU_IDS, "1");
ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false);
ResourcePtr gpu_0 = ResourceFactory::Create("gpu0", "GPU", 0);
ResourcePtr gpu_1 = ResourceFactory::Create("gpu1", "GPU", 1);
@ -142,7 +138,7 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) {
cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj);
}
TEST_F(SchedulerTest, OnLoadCompleted) {
TEST_F(SchedulerTest, ON_LOAD_COMPLETED) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = std::make_shared<meta::TableFileSchema>();
@ -162,7 +158,7 @@ TEST_F(SchedulerTest, OnLoadCompleted) {
}
TEST_F(SchedulerTest, PushTaskToNeighbourRandomlyTest) {
TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy1 = std::make_shared<meta::TableFileSchema>();
@ -233,7 +229,7 @@ protected:
};
TEST_F(SchedulerTest2, SpecifiedResourceTest) {
TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = std::make_shared<meta::TableFileSchema>();

2
cpp/unittest/scheduler/task_test.cpp
View File

@ -13,7 +13,7 @@ namespace milvus {
namespace engine {
TEST(TaskTest, invalid_index) {
TEST(TaskTest, INVALID_INDEX) {
auto search_task = std::make_shared<XSearchTask>(nullptr);
search_task->Load(LoadType::TEST, 10);
}

60
cpp/unittest/scheduler/tasktable_test.cpp
View File

@ -32,18 +32,18 @@ protected:
std::vector<TaskTableItemPtr> items_;
};
TEST_F(TaskTableItemTest, construct) {
TEST_F(TaskTableItemTest, CONSTRUCT) {
ASSERT_EQ(default_.id, 0);
ASSERT_EQ(default_.task, nullptr);
ASSERT_EQ(default_.state, TaskTableItemState::INVALID);
}
TEST_F(TaskTableItemTest, destruct) {
TEST_F(TaskTableItemTest, DESTRUCT) {
auto p_item = new TaskTableItem();
delete p_item;
}
TEST_F(TaskTableItemTest, is_finish) {
TEST_F(TaskTableItemTest, IS_FINISH) {
for (auto &item : items_) {
if (item->state == TaskTableItemState::EXECUTED
|| item->state == TaskTableItemState::MOVED) {
@ -54,13 +54,13 @@ TEST_F(TaskTableItemTest, is_finish) {
}
}
TEST_F(TaskTableItemTest, dump) {
TEST_F(TaskTableItemTest, DUMP) {
for (auto &item : items_) {
ASSERT_FALSE(item->Dump().empty());
}
}
TEST_F(TaskTableItemTest, load) {
TEST_F(TaskTableItemTest, LOAD) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Load();
@ -74,7 +74,7 @@ TEST_F(TaskTableItemTest, load) {
}
}
TEST_F(TaskTableItemTest, loaded) {
TEST_F(TaskTableItemTest, LOADED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Loaded();
@ -88,7 +88,7 @@ TEST_F(TaskTableItemTest, loaded) {
}
}
TEST_F(TaskTableItemTest, execute) {
TEST_F(TaskTableItemTest, EXECUTE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Execute();
@ -103,7 +103,7 @@ TEST_F(TaskTableItemTest, execute) {
}
TEST_F(TaskTableItemTest, executed) {
TEST_F(TaskTableItemTest, EXECUTED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Executed();
@ -117,7 +117,7 @@ TEST_F(TaskTableItemTest, executed) {
}
}
TEST_F(TaskTableItemTest, move) {
TEST_F(TaskTableItemTest, MOVE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Move();
@ -131,7 +131,7 @@ TEST_F(TaskTableItemTest, move) {
}
}
TEST_F(TaskTableItemTest, moved) {
TEST_F(TaskTableItemTest, MOVED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Moved();
@ -163,7 +163,7 @@ protected:
TaskTable empty_table_;
};
TEST_F(TaskTableBaseTest, subscriber) {
TEST_F(TaskTableBaseTest, SUBSCRIBER) {
bool flag = false;
auto callback = [&]() {
flag = true;
@ -174,46 +174,46 @@ TEST_F(TaskTableBaseTest, subscriber) {
}
TEST_F(TaskTableBaseTest, put_task) {
TEST_F(TaskTableBaseTest, PUT_TASK) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
}
TEST_F(TaskTableBaseTest, put_invalid_test) {
TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) {
empty_table_.Put(invalid_task_);
ASSERT_EQ(empty_table_.Get(0)->task, invalid_task_);
}
TEST_F(TaskTableBaseTest, put_batch) {
TEST_F(TaskTableBaseTest, PUT_BATCH) {
std::vector<TaskPtr> tasks{task1_, task2_};
empty_table_.Put(tasks);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
ASSERT_EQ(empty_table_.Get(1)->task, task2_);
}
TEST_F(TaskTableBaseTest, put_empty_batch) {
TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) {
std::vector<TaskPtr> tasks{};
empty_table_.Put(tasks);
}
TEST_F(TaskTableBaseTest, empty) {
TEST_F(TaskTableBaseTest, EMPTY) {
ASSERT_TRUE(empty_table_.Empty());
empty_table_.Put(task1_);
ASSERT_FALSE(empty_table_.Empty());
}
TEST_F(TaskTableBaseTest, size) {
TEST_F(TaskTableBaseTest, SIZE) {
ASSERT_EQ(empty_table_.Size(), 0);
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Size(), 1);
}
TEST_F(TaskTableBaseTest, operator_) {
TEST_F(TaskTableBaseTest, OPERATOR) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0), empty_table_[0]);
}
TEST_F(TaskTableBaseTest, pick_to_load) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -226,7 +226,7 @@ TEST_F(TaskTableBaseTest, pick_to_load) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_load_limit) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -241,7 +241,7 @@ TEST_F(TaskTableBaseTest, pick_to_load_limit) {
ASSERT_EQ(indexes[2], 4);
}
TEST_F(TaskTableBaseTest, pick_to_load_cache) {
TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -262,7 +262,7 @@ TEST_F(TaskTableBaseTest, pick_to_load_cache) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_execute) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -276,7 +276,7 @@ TEST_F(TaskTableBaseTest, pick_to_execute) {
ASSERT_EQ(indexes[0], 2);
}
TEST_F(TaskTableBaseTest, pick_to_execute_limit) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -292,7 +292,7 @@ TEST_F(TaskTableBaseTest, pick_to_execute_limit) {
ASSERT_EQ(indexes[1], 3);
}
TEST_F(TaskTableBaseTest, pick_to_execute_cache) {
TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
const size_t NUM_TASKS = 10;
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
@ -340,7 +340,7 @@ protected:
TaskTable table1_;
};
TEST_F(TaskTableAdvanceTest, load) {
TEST_F(TaskTableAdvanceTest, LOAD) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
@ -359,7 +359,7 @@ TEST_F(TaskTableAdvanceTest, load) {
}
}
TEST_F(TaskTableAdvanceTest, loaded) {
TEST_F(TaskTableAdvanceTest, LOADED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
@ -378,7 +378,7 @@ TEST_F(TaskTableAdvanceTest, loaded) {
}
}
TEST_F(TaskTableAdvanceTest, execute) {
TEST_F(TaskTableAdvanceTest, EXECUTE) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
@ -397,7 +397,7 @@ TEST_F(TaskTableAdvanceTest, execute) {
}
}
TEST_F(TaskTableAdvanceTest, executed) {
TEST_F(TaskTableAdvanceTest, EXECUTED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
@ -416,7 +416,7 @@ TEST_F(TaskTableAdvanceTest, executed) {
}
}
TEST_F(TaskTableAdvanceTest, move) {
TEST_F(TaskTableAdvanceTest, MOVE) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
@ -435,7 +435,7 @@ TEST_F(TaskTableAdvanceTest, move) {
}
}
TEST_F(TaskTableAdvanceTest, moved) {
TEST_F(TaskTableAdvanceTest, MOVED) {
std::vector<TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);

5
cpp/unittest/server/appendix/server_config.yaml
View File

@ -1,8 +1,8 @@
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
db_config:
db_path: /tmp/milvus # milvus data storage path
@ -17,6 +17,7 @@ db_config:
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
metric_config:
is_startup: off # if monitoring start: on, off
@ -32,8 +33,6 @@ cache_config:
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
gpu_ids: # gpu id
- 0
engine_config:
use_blas_threshold: 20

3
cpp/unittest/server/cache_test.cpp
View File

@ -166,9 +166,6 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
TEST(CacheTest, GPU_CACHE_TEST) {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(0);
const int dim = 256;

4
cpp/unittest/server/config_test.cpp
View File

@ -144,8 +144,4 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
err = config.ValidateConfig();
ASSERT_NE(err, SERVER_SUCCESS);
server_config.SetValue(server::CONFIG_GPU_INDEX, "9999");
err = config.ValidateConfig();
ASSERT_NE(err, SERVER_SUCCESS);
}