From defd24b2c6d644c754394749338bcaae1d6c117a Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 10 Oct 2019 16:03:29 +0800 Subject: [PATCH 1/6] format code Former-commit-id: 648cc3be7f2dc4cca5c1394ff83c2d383c7b70b7 --- cpp/src/core/unittest/kdtree.cpp | 11 +- cpp/src/core/unittest/test_idmap.cpp | 53 +++-- cpp/src/core/unittest/test_ivf.cpp | 146 +++++++------- cpp/src/core/unittest/test_json.cpp | 8 +- cpp/src/core/unittest/test_kdt.cpp | 24 +-- cpp/src/core/unittest/test_nsg/test_nsg.cpp | 30 ++- cpp/src/core/unittest/utils.cpp | 39 ++-- cpp/unittest/db/test_db.cpp | 124 ++++++------ cpp/unittest/db/test_db_mysql.cpp | 44 ++--- cpp/unittest/db/test_engine.cpp | 42 ++-- cpp/unittest/db/test_mem.cpp | 101 +++++----- cpp/unittest/db/test_meta.cpp | 126 ++++++------ cpp/unittest/db/test_meta_mysql.cpp | 133 ++++++------- cpp/unittest/db/test_misc.cpp | 42 ++-- cpp/unittest/db/test_search.cpp | 78 ++++---- cpp/unittest/db/utils.cpp | 50 +++-- cpp/unittest/metrics/test_metricbase.cpp | 10 +- cpp/unittest/metrics/test_metrics.cpp | 62 +++--- cpp/unittest/metrics/test_prometheus.cpp | 14 +- cpp/unittest/metrics/utils.cpp | 8 +- cpp/unittest/scheduler/test_tasktable.cpp | 178 ++++++++--------- cpp/unittest/server/test_cache.cpp | 72 ++++--- cpp/unittest/server/test_config.cpp | 34 ++-- cpp/unittest/server/test_rpc.cpp | 72 ++++--- cpp/unittest/server/util_test.cpp | 208 ++++++++++---------- cpp/unittest/wrapper/test_wrapper.cpp | 68 ++++--- 26 files changed, 850 insertions(+), 927 deletions(-) diff --git a/cpp/src/core/unittest/kdtree.cpp b/cpp/src/core/unittest/kdtree.cpp index 9b1e428a80..6b209919ef 100644 --- a/cpp/src/core/unittest/kdtree.cpp +++ b/cpp/src/core/unittest/kdtree.cpp @@ -22,13 +22,8 @@ //#include "knowhere/index/vector_index/cpu_kdt_rng.h" //#include "knowhere/index/vector_index/definitions.h" // -// namespace { // -// namespace kn = knowhere; -// -//} // namespace -// -// kn::DatasetPtr +// knowhere::DatasetPtr // generate_dataset(int64_t n, int64_t d, int64_t base) { // auto elems = n * d; // auto p_data = (float*)malloc(elems * sizeof(float)); @@ -58,7 +53,7 @@ // return dataset; //} // -// kn::DatasetPtr +// knowhere::DatasetPtr // generate_queries(int64_t n, int64_t d, int64_t k, int64_t base) { // size_t size = sizeof(float) * n * d; // auto v = (float*)malloc(size); @@ -86,7 +81,7 @@ // std::vector fields{field}; // auto schema = std::make_shared(fields); // -// return std::make_shared(data, schema); +// return std::make_shared(data, schema); //} // // int diff --git a/cpp/src/core/unittest/test_idmap.cpp b/cpp/src/core/unittest/test_idmap.cpp index e907d309eb..077f7328f7 100644 --- a/cpp/src/core/unittest/test_idmap.cpp +++ b/cpp/src/core/unittest/test_idmap.cpp @@ -25,33 +25,27 @@ #include "unittest/utils.h" -namespace { - -namespace kn = knowhere; - -} // namespace - static int device_id = 0; class IDMAPTest : public DataGen, public ::testing::Test { protected: void SetUp() override { - kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); Init_with_default(); - index_ = std::make_shared(); + index_ = std::make_shared(); } void TearDown() override { - kn::FaissGpuResourceMgr::GetInstance().Free(); + knowhere::FaissGpuResourceMgr::GetInstance().Free(); } protected: - kn::IDMAPPtr index_ = nullptr; + knowhere::IDMAPPtr index_ = nullptr; }; void -AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { +AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); @@ -59,7 +53,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { } void -PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { +PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -80,10 +74,10 @@ PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { TEST_F(IDMAPTest, idmap_basic) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; index_->Train(conf); index_->Add(base_dataset, conf); @@ -97,7 +91,7 @@ TEST_F(IDMAPTest, idmap_basic) { index_->Seal(); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); @@ -105,7 +99,7 @@ TEST_F(IDMAPTest, idmap_basic) { } TEST_F(IDMAPTest, idmap_serialize) { - auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) { + auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) { FileIOWriter writer(filename); writer(static_cast(bin->data.get()), bin->size); @@ -113,15 +107,15 @@ TEST_F(IDMAPTest, idmap_serialize) { reader(ret, bin->size); }; - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; { // serialize index index_->Train(conf); - index_->Add(base_dataset, kn::Config()); + index_->Add(base_dataset, knowhere::Config()); auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); PrintResult(re_result, nq, k); @@ -151,10 +145,10 @@ TEST_F(IDMAPTest, idmap_serialize) { TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; index_->Train(conf); index_->Add(base_dataset, conf); @@ -175,12 +169,12 @@ TEST_F(IDMAPTest, copy_test) { { // cpu to gpu - auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf); + auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, conf); auto clone_result = clone_index->Search(query_dataset, conf); AssertAnns(clone_result, nq, k); - ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawVectors(); }, + ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawVectors(); }, knowhere::KnowhereException); - ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawIds(); }, + ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawIds(); }, knowhere::KnowhereException); auto binary = clone_index->Serialize(); @@ -193,15 +187,16 @@ TEST_F(IDMAPTest, copy_test) { AssertAnns(clone_gpu_res, nq, k); // gpu to cpu - auto host_index = kn::cloner::CopyGpuToCpu(clone_index, conf); + auto host_index = knowhere::cloner::CopyGpuToCpu(clone_index, conf); auto host_result = host_index->Search(query_dataset, conf); AssertAnns(host_result, nq, k); - ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawVectors() != nullptr); - ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawIds() != nullptr); + ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawVectors() != nullptr); + ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawIds() != nullptr); // gpu to gpu - auto device_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf); - auto new_device_index = std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, conf); + auto device_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, conf); + auto new_device_index = + std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, conf); auto device_result = new_device_index->Search(query_dataset, conf); AssertAnns(device_result, nq, k); } diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index 21e658cec0..4e309c3867 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -37,12 +37,6 @@ #include "unittest/utils.h" -namespace { - -namespace kn = knowhere; - -} // namespace - using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; @@ -53,22 +47,22 @@ constexpr int64_t NB = 1000000 / 100; constexpr int64_t NQ = 10; constexpr int64_t K = 10; -kn::IVFIndexPtr +knowhere::IVFIndexPtr IndexFactory(const std::string& type) { if (type == "IVF") { - return std::make_shared(); + return std::make_shared(); } else if (type == "IVFPQ") { - return std::make_shared(); + return std::make_shared(); } else if (type == "GPUIVF") { - return std::make_shared(device_id); + return std::make_shared(device_id); } else if (type == "GPUIVFPQ") { - return std::make_shared(device_id); + return std::make_shared(device_id); } else if (type == "IVFSQ") { - return std::make_shared(); + return std::make_shared(); } else if (type == "GPUIVFSQ") { - return std::make_shared(device_id); + return std::make_shared(device_id); } else if (type == "IVFSQHybrid") { - return std::make_shared(device_id); + return std::make_shared(device_id); } } @@ -88,19 +82,19 @@ class ParamGenerator { return instance; } - kn::Config + knowhere::Config Gen(const ParameterType& type) { if (type == ParameterType::ivf) { - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; tempconf->nprobe = 16; tempconf->k = K; - tempconf->metric_type = kn::METRICTYPE::L2; + tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } else if (type == ParameterType::ivfpq) { - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; @@ -108,17 +102,17 @@ class ParamGenerator { tempconf->k = K; tempconf->m = 8; tempconf->nbits = 8; - tempconf->metric_type = kn::METRICTYPE::L2; + tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) { - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; tempconf->nprobe = 16; tempconf->k = K; tempconf->nbits = 8; - tempconf->metric_type = kn::METRICTYPE::L2; + tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } } @@ -134,28 +128,28 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple gpu_idx{"GPUIVFSQ"}; auto finder = std::find(gpu_idx.cbegin(), gpu_idx.cend(), index_type); if (finder != gpu_idx.cend()) { - return kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config()); + return knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config()); } return index_; } protected: std::string index_type; - kn::Config conf; - kn::IVFIndexPtr index_ = nullptr; + knowhere::Config conf; + knowhere::IVFIndexPtr index_ = nullptr; }; INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, @@ -168,7 +162,7 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid))); void -AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { +AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); @@ -176,7 +170,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { } void -PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { +PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -232,12 +226,12 @@ TEST_P(IVFTest, hybrid) { // AssertAnns(result, nq, conf->k); { - auto hybrid_1_idx = std::make_shared(device_id); + auto hybrid_1_idx = std::make_shared(device_id); auto binaryset = index_->Serialize(); hybrid_1_idx->Load(binaryset); - auto quantizer_conf = std::make_shared(); + auto quantizer_conf = std::make_shared(); quantizer_conf->mode = 1; quantizer_conf->gpu_id = device_id; auto q = hybrid_1_idx->LoadQuantizer(quantizer_conf); @@ -248,12 +242,12 @@ TEST_P(IVFTest, hybrid) { } { - auto hybrid_2_idx = std::make_shared(device_id); + auto hybrid_2_idx = std::make_shared(device_id); auto binaryset = index_->Serialize(); hybrid_2_idx->Load(binaryset); - auto quantizer_conf = std::make_shared(); + auto quantizer_conf = std::make_shared(); quantizer_conf->mode = 1; quantizer_conf->gpu_id = device_id; auto q = hybrid_2_idx->LoadQuantizer(quantizer_conf); @@ -291,7 +285,7 @@ TEST_P(IVFTest, hybrid) { //} TEST_P(IVFTest, ivf_serialize) { - auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) { + auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) { FileIOWriter writer(filename); writer(static_cast(bin->data.get()), bin->size); @@ -365,7 +359,7 @@ TEST_P(IVFTest, clone_test) { AssertAnns(result, nq, conf->k); // PrintResult(result, nq, k); - auto AssertEqual = [&](kn::DatasetPtr p1, kn::DatasetPtr p2) { + auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) { auto ids_p1 = p1->array()[0]; auto ids_p2 = p2->array()[0]; @@ -406,7 +400,7 @@ TEST_P(IVFTest, clone_test) { auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ - auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); auto clone_result = clone_index->Search(query_dataset, conf); AssertEqual(result, clone_result); std::cout << "clone G <=> C [" << index_type << "] success" << std::endl; @@ -415,9 +409,9 @@ TEST_P(IVFTest, clone_test) { EXPECT_THROW( { std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl; - auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); }, - kn::KnowhereException); + knowhere::KnowhereException); } } @@ -427,7 +421,7 @@ TEST_P(IVFTest, clone_test) { auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ - auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config()); + auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config()); auto clone_result = clone_index->Search(query_dataset, conf); AssertEqual(result, clone_result); std::cout << "clone C <=> G [" << index_type << "] success" << std::endl; @@ -436,9 +430,9 @@ TEST_P(IVFTest, clone_test) { EXPECT_THROW( { std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl; - auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config()); + auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config()); }, - kn::KnowhereException); + knowhere::KnowhereException); } } } @@ -466,14 +460,14 @@ TEST_P(IVFTest, seal_test) { auto result = new_idx->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); - auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); - kn::TimeRecorder tc("CopyToGpu"); - kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + knowhere::TimeRecorder tc("CopyToGpu"); + knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); auto without_seal = tc.RecordSection("Without seal"); cpu_idx->Seal(); tc.RecordSection("seal cost"); - kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); auto with_seal = tc.RecordSection("With seal"); ASSERT_GE(without_seal, with_seal); } @@ -483,7 +477,7 @@ class GPURESTEST : public DataGen, public ::testing::Test { void SetUp() override { Generate(128, 1000000, 1000); - kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); k = 100; elems = nq * k; @@ -495,12 +489,12 @@ class GPURESTEST : public DataGen, public ::testing::Test { TearDown() override { delete ids; delete dis; - kn::FaissGpuResourceMgr::GetInstance().Free(); + knowhere::FaissGpuResourceMgr::GetInstance().Free(); } protected: std::string index_type; - kn::IVFIndexPtr index_ = nullptr; + knowhere::IVFIndexPtr index_ = nullptr; int64_t* ids = nullptr; float* dis = nullptr; @@ -514,16 +508,16 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { assert(!xb.empty()); { - index_ = std::make_shared(-1); - ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), -1); - std::dynamic_pointer_cast(index_)->SetGpuDevice(device_id); - ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), device_id); + index_ = std::make_shared(-1); + ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), -1); + std::dynamic_pointer_cast(index_)->SetGpuDevice(device_id); + ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), device_id); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; conf->k = k; conf->nprobe = 1; @@ -535,7 +529,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - kn::TimeRecorder tc("knowere GPUIVF"); + knowhere::TimeRecorder tc("knowere GPUIVF"); for (int i = 0; i < search_count; ++i) { index_->Search(query_dataset, conf); if (i > search_count - 6 || i < 5) @@ -543,7 +537,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { } tc.ElapseFromBegin("search all"); } - kn::FaissGpuResourceMgr::GetInstance().Dump(); + knowhere::FaissGpuResourceMgr::GetInstance().Dump(); { // IVF-Search @@ -554,7 +548,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { device_index.train(nb, xb.data()); device_index.add(nb, xb.data()); - kn::TimeRecorder tc("ori IVF"); + knowhere::TimeRecorder tc("ori IVF"); for (int i = 0; i < search_count; ++i) { device_index.search(nq, xq.data(), k, dis, ids); if (i > search_count - 6 || i < 5) @@ -570,11 +564,11 @@ TEST_F(GPURESTEST, gpuivfsq) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -587,11 +581,11 @@ TEST_F(GPURESTEST, gpuivfsq) { // auto result = index_->Search(query_dataset, conf); // AssertAnns(result, nq, k); - auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); cpu_idx->Seal(); - kn::TimeRecorder tc("knowhere GPUSQ8"); - auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + knowhere::TimeRecorder tc("knowhere GPUSQ8"); + auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); tc.RecordSection("Copy to gpu"); for (int i = 0; i < search_count; ++i) { search_idx->Search(query_dataset, conf); @@ -622,7 +616,7 @@ TEST_F(GPURESTEST, gpuivfsq) { faiss::gpu::GpuClonerOptions option; option.allInGpu = true; - kn::TimeRecorder tc("ori GPUSQ8"); + knowhere::TimeRecorder tc("ori GPUSQ8"); faiss::Index* search_idx = faiss::gpu::index_cpu_to_gpu(&res, device_id, cpu_index, &option); tc.RecordSection("Copy to gpu"); for (int i = 0; i < search_count; ++i) { @@ -643,11 +637,11 @@ TEST_F(GPURESTEST, copyandsearch) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -660,10 +654,10 @@ TEST_F(GPURESTEST, copyandsearch) { // auto result = index_->Search(query_dataset, conf); // AssertAnns(result, nq, k); - auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config()); cpu_idx->Seal(); - auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); auto search_func = [&] { // TimeRecorder tc("search&load"); @@ -677,15 +671,15 @@ TEST_F(GPURESTEST, copyandsearch) { auto load_func = [&] { // TimeRecorder tc("search&load"); for (int i = 0; i < load_count; ++i) { - kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); // if (i > load_count -5 || i < 5) // tc.RecordSection("Copy to gpu"); } // tc.ElapseFromBegin("load finish"); }; - kn::TimeRecorder tc("basic"); - kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + knowhere::TimeRecorder tc("basic"); + knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); tc.RecordSection("Copy to gpu once"); search_idx->Search(query_dataset, conf); tc.RecordSection("search once"); @@ -705,11 +699,11 @@ TEST_F(GPURESTEST, TrainAndSearch) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = kn::METRICTYPE::L2; + conf->metric_type = knowhere::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -720,9 +714,9 @@ TEST_F(GPURESTEST, TrainAndSearch) { auto new_index = IndexFactory(index_type); new_index->set_index_model(model); new_index->Add(base_dataset, conf); - auto cpu_idx = kn::cloner::CopyGpuToCpu(new_index, kn::Config()); + auto cpu_idx = knowhere::cloner::CopyGpuToCpu(new_index, knowhere::Config()); cpu_idx->Seal(); - auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); constexpr int train_count = 1; constexpr int search_count = 5000; @@ -734,7 +728,7 @@ TEST_F(GPURESTEST, TrainAndSearch) { test_idx->Add(base_dataset, conf); } }; - auto search_stage = [&](kn::VectorIndexPtr& search_idx) { + auto search_stage = [&](knowhere::VectorIndexPtr& search_idx) { for (int i = 0; i < search_count; ++i) { auto result = search_idx->Search(query_dataset, conf); AssertAnns(result, nq, k); @@ -763,7 +757,7 @@ TEST_F(GPURESTEST, TrainAndSearch) { } { // search parallel - auto search_idx_2 = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + auto search_idx_2 = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config()); std::thread search_1(search_stage, std::ref(search_idx)); std::thread search_2(search_stage, std::ref(search_idx_2)); search_1.join(); diff --git a/cpp/src/core/unittest/test_json.cpp b/cpp/src/core/unittest/test_json.cpp index 613e0deff5..a3babbc1b7 100644 --- a/cpp/src/core/unittest/test_json.cpp +++ b/cpp/src/core/unittest/test_json.cpp @@ -17,15 +17,9 @@ #include "knowhere/common/config.h" -namespace { - -namespace kn = knowhere; - -} // namespace - int main() { - kn::Config cfg; + knowhere::Config cfg; cfg["size"] = size_t(199); auto size = cfg.get_with_default("size", 123); diff --git a/cpp/src/core/unittest/test_kdt.cpp b/cpp/src/core/unittest/test_kdt.cpp index f9e02bd9a4..875944be83 100644 --- a/cpp/src/core/unittest/test_kdt.cpp +++ b/cpp/src/core/unittest/test_kdt.cpp @@ -28,12 +28,6 @@ #include "unittest/utils.h" -namespace { - -namespace kn = knowhere; - -} // namespace - using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; @@ -42,9 +36,9 @@ class KDTTest : public DataGen, public ::testing::Test { protected: void SetUp() override { - index_ = std::make_shared(); + index_ = std::make_shared(); - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->tptnubmber = 1; tempconf->k = 10; conf = tempconf; @@ -53,12 +47,12 @@ class KDTTest : public DataGen, public ::testing::Test { } protected: - kn::Config conf; - std::shared_ptr index_ = nullptr; + knowhere::Config conf; + std::shared_ptr index_ = nullptr; }; void -AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { +AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); @@ -66,7 +60,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { } void -PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { +PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -125,7 +119,7 @@ TEST_F(KDTTest, kdt_serialize) { auto model = index_->Train(base_dataset, conf); // index_->Add(base_dataset, conf); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); @@ -150,7 +144,7 @@ TEST_F(KDTTest, kdt_serialize) { ++fileno; } - kn::BinarySet load_data_list; + knowhere::BinarySet load_data_list; for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { auto bin_size = meta_list[i].second; FileIOReader reader(filename_list[i]); @@ -162,7 +156,7 @@ TEST_F(KDTTest, kdt_serialize) { load_data_list.Append(meta_list[i].first, data, bin_size); } - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(load_data_list); auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); diff --git a/cpp/src/core/unittest/test_nsg/test_nsg.cpp b/cpp/src/core/unittest/test_nsg/test_nsg.cpp index b59f0d4928..5aaa65abe2 100644 --- a/cpp/src/core/unittest/test_nsg/test_nsg.cpp +++ b/cpp/src/core/unittest/test_nsg/test_nsg.cpp @@ -26,12 +26,6 @@ #include "unittest/utils.h" -namespace { - -namespace kn = knowhere; - -} // namespace - using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; @@ -43,11 +37,11 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { void SetUp() override { // Init_with_default(); - kn::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); Generate(256, 1000000, 1); - index_ = std::make_shared(); + index_ = std::make_shared(); - auto tmp_conf = std::make_shared(); + auto tmp_conf = std::make_shared(); tmp_conf->gpu_id = DEVICE_ID; tmp_conf->knng = 100; tmp_conf->nprobe = 32; @@ -55,10 +49,10 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { tmp_conf->search_length = 60; tmp_conf->out_degree = 70; tmp_conf->candidate_pool_size = 500; - tmp_conf->metric_type = kn::METRICTYPE::L2; + tmp_conf->metric_type = knowhere::METRICTYPE::L2; train_conf = tmp_conf; - auto tmp2_conf = std::make_shared(); + auto tmp2_conf = std::make_shared(); tmp2_conf->k = k; tmp2_conf->search_length = 30; search_conf = tmp2_conf; @@ -66,17 +60,17 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { void TearDown() override { - kn::FaissGpuResourceMgr::GetInstance().Free(); + knowhere::FaissGpuResourceMgr::GetInstance().Free(); } protected: - std::shared_ptr index_; - kn::Config train_conf; - kn::Config search_conf; + std::shared_ptr index_; + knowhere::Config train_conf; + knowhere::Config search_conf; }; void -AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { +AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); @@ -91,7 +85,7 @@ TEST_F(NSGInterfaceTest, basic_test) { AssertAnns(result, nq, k); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto new_result = new_index->Search(query_dataset, search_conf); AssertAnns(result, nq, k); @@ -100,7 +94,7 @@ TEST_F(NSGInterfaceTest, basic_test) { ASSERT_EQ(index_->Dimension(), dim); ASSERT_THROW({ index_->Clone(); }, knowhere::KnowhereException); ASSERT_NO_THROW({ - index_->Add(base_dataset, kn::Config()); + index_->Add(base_dataset, knowhere::Config()); index_->Seal(); }); diff --git a/cpp/src/core/unittest/utils.cpp b/cpp/src/core/unittest/utils.cpp index 3c342738c1..cdfc56b1cb 100644 --- a/cpp/src/core/unittest/utils.cpp +++ b/cpp/src/core/unittest/utils.cpp @@ -23,12 +23,6 @@ INITIALIZE_EASYLOGGINGPP -namespace { - -namespace kn = knowhere; - -} // namespace - void InitLog() { el::Configurations defaultConf; @@ -124,31 +118,32 @@ FileIOWriter::operator()(void* ptr, size_t size) { return size; } -kn::DatasetPtr +knowhere::DatasetPtr generate_dataset(int64_t nb, int64_t dim, float* xb, int64_t* ids) { std::vector shape{nb, dim}; - auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{kn::ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto id_array = kn::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t)); - std::vector arrays{id_array}; - std::vector array_fields{kn::ConstructInt64Field("id")}; - auto array_schema = std::make_shared(tensor_fields); + auto id_array = knowhere::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t)); + std::vector arrays{id_array}; + std::vector array_fields{knowhere::ConstructInt64Field("id")}; + auto array_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); + auto dataset = + std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); return dataset; } -kn::DatasetPtr +knowhere::DatasetPtr generate_query_dataset(int64_t nb, int64_t dim, float* xb) { std::vector shape{nb, dim}; - auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{kn::ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(tensors), tensor_schema); + auto dataset = std::make_shared(std::move(tensors), tensor_schema); return dataset; } diff --git a/cpp/unittest/db/test_db.cpp b/cpp/unittest/db/test_db.cpp index eb1f947d61..e502d0f742 100644 --- a/cpp/unittest/db/test_db.cpp +++ b/cpp/unittest/db/test_db.cpp @@ -31,8 +31,6 @@ namespace { -namespace ms = milvus; - static const char *TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t VECTOR_COUNT = 25000; @@ -40,9 +38,9 @@ static constexpr int64_t INSERT_LOOP = 1000; static constexpr int64_t SECONDS_EACH_HOUR = 3600; static constexpr int64_t DAY_SECONDS = 24 * 60 * 60; -ms::engine::meta::TableSchema +milvus::engine::meta::TableSchema BuildTableSchema() { - ms::engine::meta::TableSchema table_info; + milvus::engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; table_info.table_id_ = TABLE_NAME; return table_info; @@ -77,7 +75,7 @@ CurrentTmDate(int64_t offset_day = 0) { void ConvertTimeRangeToDBDates(const std::string &start_value, const std::string &end_value, - std::vector &dates) { + std::vector &dates) { dates.clear(); time_t tt_start, tt_end; @@ -111,41 +109,41 @@ ConvertTimeRangeToDBDates(const std::string &start_value, TEST_F(DBTest, CONFIG_TEST) { { - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf("wrong")); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf("wrong")); /* EXPECT_DEATH(engine::ArchiveConf conf("wrong"), ""); */ } { - ms::engine::ArchiveConf conf("delete"); + milvus::engine::ArchiveConf conf("delete"); ASSERT_EQ(conf.GetType(), "delete"); auto criterias = conf.GetCriterias(); ASSERT_EQ(criterias.size(), 0); } { - ms::engine::ArchiveConf conf("swap"); + milvus::engine::ArchiveConf conf("swap"); ASSERT_EQ(conf.GetType(), "swap"); auto criterias = conf.GetCriterias(); ASSERT_EQ(criterias.size(), 0); } { - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "disk:")); - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "disk:a")); - ms::engine::ArchiveConf conf("swap", "disk:1024"); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "disk:")); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "disk:a")); + milvus::engine::ArchiveConf conf("swap", "disk:1024"); auto criterias = conf.GetCriterias(); ASSERT_EQ(criterias.size(), 1); ASSERT_EQ(criterias["disk"], 1024); } { - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:")); - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a")); - ms::engine::ArchiveConf conf("swap", "days:100"); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "days:")); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "days:a")); + milvus::engine::ArchiveConf conf("swap", "days:100"); auto criterias = conf.GetCriterias(); ASSERT_EQ(criterias.size(), 1); ASSERT_EQ(criterias["days"], 100); } { - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:")); - ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a")); - ms::engine::ArchiveConf conf("swap", "days:100;disk:200"); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "days:")); + ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "days:a")); + milvus::engine::ArchiveConf conf("swap", "days:100;disk:200"); auto criterias = conf.GetCriterias(); ASSERT_EQ(criterias.size(), 2); ASSERT_EQ(criterias["days"], 100); @@ -154,17 +152,17 @@ TEST_F(DBTest, CONFIG_TEST) { } TEST_F(DBTest, DB_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - ms::engine::IDNumbers vector_ids; - ms::engine::IDNumbers target_ids; + milvus::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers target_ids; int64_t nb = 50; std::vector xb; @@ -175,7 +173,7 @@ TEST_F(DBTest, DB_TEST) { BuildVectors(qb, qxb); std::thread search([&]() { - ms::engine::QueryResults results; + milvus::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -191,7 +189,7 @@ TEST_F(DBTest, DB_TEST) { START_TIMER; stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count / ms::engine::M << " M"; + ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -230,10 +228,10 @@ TEST_F(DBTest, DB_TEST) { } TEST_F(DBTest, SEARCH_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -274,20 +272,20 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } - ms::engine::TableIndex index; - index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; + milvus::engine::TableIndex index; + index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish { - ms::engine::QueryResults results; + milvus::engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); ASSERT_TRUE(stat.ok()); } {//search by specify index file - ms::engine::meta::DatesT dates; + milvus::engine::meta::DatesT dates; std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; - ms::engine::QueryResults results; + milvus::engine::QueryResults results; stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); ASSERT_TRUE(stat.ok()); } @@ -296,10 +294,10 @@ TEST_F(DBTest, SEARCH_TEST) { } TEST_F(DBTest, PRELOADTABLE_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -311,26 +309,26 @@ TEST_F(DBTest, PRELOADTABLE_TEST) { int loop = 5; for (auto i = 0; i < loop; ++i) { - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids.size(), nb); } - ms::engine::TableIndex index; - index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; + milvus::engine::TableIndex index; + index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish - int64_t prev_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage(); + int64_t prev_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage(); stat = db_->PreloadTable(TABLE_NAME); ASSERT_TRUE(stat.ok()); - int64_t cur_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage(); + int64_t cur_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage(); ASSERT_TRUE(prev_cache_usage < cur_cache_usage); } TEST_F(DBTest, SHUTDOWN_TEST) { db_->Stop(); - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); ASSERT_FALSE(stat.ok()); @@ -341,7 +339,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) { stat = db_->HasTable(table_info.table_id_, has_table); ASSERT_FALSE(stat.ok()); - ms::engine::IDNumbers ids; + milvus::engine::IDNumbers ids; stat = db_->InsertVectors(table_info.table_id_, 0, nullptr, ids); ASSERT_FALSE(stat.ok()); @@ -352,15 +350,15 @@ TEST_F(DBTest, SHUTDOWN_TEST) { stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_FALSE(stat.ok()); - ms::engine::TableIndex index; + milvus::engine::TableIndex index; stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_FALSE(stat.ok()); stat = db_->DescribeIndex(table_info.table_id_, index); ASSERT_FALSE(stat.ok()); - ms::engine::meta::DatesT dates; - ms::engine::QueryResults results; + milvus::engine::meta::DatesT dates; + milvus::engine::QueryResults results; stat = db_->Query(table_info.table_id_, 1, 1, 1, nullptr, dates, results); ASSERT_FALSE(stat.ok()); std::vector file_ids; @@ -372,24 +370,24 @@ TEST_F(DBTest, SHUTDOWN_TEST) { } TEST_F(DBTest, INDEX_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); int64_t nb = VECTOR_COUNT; std::vector xb; BuildVectors(nb, xb); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids.size(), nb); - ms::engine::TableIndex index; - index.engine_type_ = (int) ms::engine::EngineType::FAISS_IVFSQ8; - index.metric_type_ = (int) ms::engine::MetricType::IP; + milvus::engine::TableIndex index; + index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFSQ8; + index.metric_type_ = (int) milvus::engine::MetricType::IP; stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); - ms::engine::TableIndex index_out; + milvus::engine::TableIndex index_out; stat = db_->DescribeIndex(table_info.table_id_, index_out); ASSERT_TRUE(stat.ok()); ASSERT_EQ(index.engine_type_, index_out.engine_type_); @@ -401,10 +399,10 @@ TEST_F(DBTest, INDEX_TEST) { } TEST_F(DBTest2, ARHIVE_DISK_CHECK) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - std::vector table_schema_array; + std::vector table_schema_array; stat = db_->AllTables(table_schema_array); ASSERT_TRUE(stat.ok()); bool bfound = false; @@ -416,7 +414,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { } ASSERT_TRUE(bfound); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -431,7 +429,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { int loop = INSERT_LOOP; for (auto i = 0; i < loop; ++i) { - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -440,14 +438,14 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * ms::engine::G); + ASSERT_LE(size, 1 * milvus::engine::G); } TEST_F(DBTest2, DELETE_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -463,12 +461,12 @@ TEST_F(DBTest2, DELETE_TEST) { std::vector xb; BuildVectors(nb, xb); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ms::engine::TableIndex index; + milvus::engine::TableIndex index; stat = db_->CreateIndex(TABLE_NAME, index); - std::vector dates; + std::vector dates; stat = db_->DeleteTable(TABLE_NAME, dates); std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_TRUE(stat.ok()); @@ -478,10 +476,10 @@ TEST_F(DBTest2, DELETE_TEST) { } TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -498,15 +496,15 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { std::vector xb; BuildVectors(nb, xb); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ms::engine::TableIndex index; + milvus::engine::TableIndex index; stat = db_->CreateIndex(TABLE_NAME, index); db_->Size(size); ASSERT_NE(size, 0UL); - std::vector dates; + std::vector dates; std::string start_value = CurrentTmDate(); std::string end_value = CurrentTmDate(1); ConvertTimeRangeToDBDates(start_value, end_value, dates); diff --git a/cpp/unittest/db/test_db_mysql.cpp b/cpp/unittest/db/test_db_mysql.cpp index 3b73deb9cc..ae1da8012a 100644 --- a/cpp/unittest/db/test_db_mysql.cpp +++ b/cpp/unittest/db/test_db_mysql.cpp @@ -29,19 +29,17 @@ namespace { -namespace ms = milvus; - static const char *TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t VECTOR_COUNT = 25000; static constexpr int64_t INSERT_LOOP = 1000; -ms::engine::meta::TableSchema +milvus::engine::meta::TableSchema BuildTableSchema() { - ms::engine::meta::TableSchema table_info; + milvus::engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; + table_info.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP; return table_info; } @@ -59,17 +57,17 @@ BuildVectors(int64_t n, std::vector &vectors) { } // namespace TEST_F(MySqlDBTest, DB_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - ms::engine::IDNumbers vector_ids; - ms::engine::IDNumbers target_ids; + milvus::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers target_ids; int64_t nb = 50; std::vector xb; @@ -83,7 +81,7 @@ TEST_F(MySqlDBTest, DB_TEST) { ASSERT_EQ(target_ids.size(), qb); std::thread search([&]() { - ms::engine::QueryResults results; + milvus::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(5)); @@ -99,7 +97,7 @@ TEST_F(MySqlDBTest, DB_TEST) { START_TIMER; stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count / ms::engine::M << " M"; + ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -144,10 +142,10 @@ TEST_F(MySqlDBTest, DB_TEST) { } TEST_F(MySqlDBTest, SEARCH_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -190,16 +188,16 @@ TEST_F(MySqlDBTest, SEARCH_TEST) { sleep(2); // wait until build index finish - ms::engine::QueryResults results; + milvus::engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); ASSERT_TRUE(stat.ok()); } TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - std::vector table_schema_array; + std::vector table_schema_array; stat = db_->AllTables(table_schema_array); ASSERT_TRUE(stat.ok()); bool bfound = false; @@ -211,14 +209,14 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { } ASSERT_TRUE(bfound); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - ms::engine::IDNumbers vector_ids; - ms::engine::IDNumbers target_ids; + milvus::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers target_ids; uint64_t size; db_->Size(size); @@ -237,15 +235,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * ms::engine::G); + ASSERT_LE(size, 1 * milvus::engine::G); } TEST_F(MySqlDBTest, DELETE_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); // std::cout << stat.ToString() << std::endl; - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -254,7 +252,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) { db_->HasTable(TABLE_NAME, has_table); ASSERT_TRUE(has_table); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; uint64_t size; db_->Size(size); diff --git a/cpp/unittest/db/test_engine.cpp b/cpp/unittest/db/test_engine.cpp index 40afbb38f9..137612bcab 100644 --- a/cpp/unittest/db/test_engine.cpp +++ b/cpp/unittest/db/test_engine.cpp @@ -23,63 +23,57 @@ #include "db/engine/ExecutionEngineImpl.h" #include "db/utils.h" -namespace { - -namespace ms = milvus; - -} - TEST_F(EngineTest, FACTORY_TEST) { { - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - ms::engine::EngineType::INVALID, - ms::engine::MetricType::IP, + milvus::engine::EngineType::INVALID, + milvus::engine::MetricType::IP, 1024); ASSERT_TRUE(engine_ptr == nullptr); } { - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - ms::engine::EngineType::FAISS_IDMAP, - ms::engine::MetricType::IP, + milvus::engine::EngineType::FAISS_IDMAP, + milvus::engine::MetricType::IP, 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - ms::engine::EngineType::FAISS_IVFFLAT, - ms::engine::MetricType::IP, + milvus::engine::EngineType::FAISS_IVFFLAT, + milvus::engine::MetricType::IP, 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - ms::engine::EngineType::FAISS_IVFSQ8, - ms::engine::MetricType::IP, + milvus::engine::EngineType::FAISS_IVFSQ8, + milvus::engine::MetricType::IP, 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - ms::engine::EngineType::NSG_MIX, - ms::engine::MetricType::IP, + milvus::engine::EngineType::NSG_MIX, + milvus::engine::MetricType::IP, 1024); ASSERT_TRUE(engine_ptr != nullptr); @@ -89,11 +83,11 @@ TEST_F(EngineTest, FACTORY_TEST) { TEST_F(EngineTest, ENGINE_IMPL_TEST) { uint16_t dimension = 64; std::string file_path = "/tmp/milvus_index_1"; - auto engine_ptr = ms::engine::EngineFactory::Build( + auto engine_ptr = milvus::engine::EngineFactory::Build( dimension, file_path, - ms::engine::EngineType::FAISS_IVFFLAT, - ms::engine::MetricType::IP, + milvus::engine::EngineType::FAISS_IVFFLAT, + milvus::engine::MetricType::IP, 1024); std::vector data; diff --git a/cpp/unittest/db/test_mem.cpp b/cpp/unittest/db/test_mem.cpp index 1e465a69fb..e05811ff9e 100644 --- a/cpp/unittest/db/test_mem.cpp +++ b/cpp/unittest/db/test_mem.cpp @@ -37,8 +37,6 @@ namespace { -namespace ms = milvus; - static constexpr int64_t TABLE_DIM = 256; std::string @@ -50,20 +48,20 @@ GetTableName() { return table_name; } -ms::engine::meta::TableSchema +milvus::engine::meta::TableSchema BuildTableSchema() { - ms::engine::meta::TableSchema table_info; + milvus::engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; table_info.table_id_ = GetTableName(); - table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; + table_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP; return table_info; } void -BuildVectors(int64_t n, std::vector &vectors) { +BuildVectors(int64_t n, std::vector& vectors) { vectors.clear(); vectors.resize(n * TABLE_DIM); - float *data = vectors.data(); + float* data = vectors.data(); for (int i = 0; i < n; i++) { for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); @@ -72,11 +70,11 @@ BuildVectors(int64_t n, std::vector &vectors) { } // namespace TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { - ms::engine::meta::TableSchema table_schema = BuildTableSchema(); + milvus::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - ms::engine::meta::TableFileSchema table_file_schema; + milvus::engine::meta::TableFileSchema table_file_schema; table_file_schema.table_id_ = GetTableName(); status = impl_->CreateTableFile(table_file_schema); ASSERT_TRUE(status.ok()); @@ -85,17 +83,17 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { std::vector vectors; BuildVectors(n, vectors); - ms::engine::VectorSource source(n, vectors.data()); + milvus::engine::VectorSource source(n, vectors.data()); size_t num_vectors_added; - ms::engine::ExecutionEnginePtr execution_engine_ = - ms::engine::EngineFactory::Build(table_file_schema.dimension_, - table_file_schema.location_, - (ms::engine::EngineType) table_file_schema.engine_type_, - (ms::engine::MetricType) table_file_schema.metric_type_, - table_schema.nlist_); + milvus::engine::ExecutionEnginePtr execution_engine_ = + milvus::engine::EngineFactory::Build(table_file_schema.dimension_, + table_file_schema.location_, + (milvus::engine::EngineType)table_file_schema.engine_type_, + (milvus::engine::MetricType)table_file_schema.metric_type_, + table_schema.nlist_); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids); ASSERT_TRUE(status.ok()); vector_ids = source.GetVectorIds(); @@ -115,19 +113,19 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { auto options = GetOptions(); - ms::engine::meta::TableSchema table_schema = BuildTableSchema(); + milvus::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - ms::engine::MemTableFile mem_table_file(GetTableName(), impl_, options); + milvus::engine::MemTableFile mem_table_file(GetTableName(), impl_, options); int64_t n_100 = 100; std::vector vectors_100; BuildVectors(n_100, vectors_100); - ms::engine::VectorSourcePtr source = std::make_shared(n_100, vectors_100.data()); + milvus::engine::VectorSourcePtr source = std::make_shared(n_100, vectors_100.data()); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; status = mem_table_file.Add(source, vector_ids); ASSERT_TRUE(status.ok()); @@ -139,11 +137,12 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { size_t singleVectorMem = sizeof(float) * TABLE_DIM; ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem); - int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem; + int64_t n_max = milvus::engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); - ms::engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); + milvus::engine::VectorSourcePtr + source_128M = std::make_shared(n_max, vectors_128M.data()); vector_ids.clear(); status = mem_table_file.Add(source_128M, vector_ids); @@ -156,7 +155,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_TEST) { auto options = GetOptions(); - ms::engine::meta::TableSchema table_schema = BuildTableSchema(); + milvus::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); @@ -164,27 +163,29 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_100; BuildVectors(n_100, vectors_100); - ms::engine::VectorSourcePtr source_100 = std::make_shared(n_100, vectors_100.data()); + milvus::engine::VectorSourcePtr + source_100 = std::make_shared(n_100, vectors_100.data()); - ms::engine::MemTable mem_table(GetTableName(), impl_, options); + milvus::engine::MemTable mem_table(GetTableName(), impl_, options); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; status = mem_table.Add(source_100, vector_ids); ASSERT_TRUE(status.ok()); vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); - ms::engine::MemTableFilePtr mem_table_file; + milvus::engine::MemTableFilePtr mem_table_file; mem_table.GetCurrentMemTableFile(mem_table_file); size_t singleVectorMem = sizeof(float) * TABLE_DIM; ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); - int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem; + int64_t n_max = milvus::engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); vector_ids.clear(); - ms::engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); + milvus::engine::VectorSourcePtr + source_128M = std::make_shared(n_max, vectors_128M.data()); status = mem_table.Add(source_128M, vector_ids); ASSERT_TRUE(status.ok()); @@ -200,7 +201,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_1G; BuildVectors(n_1G, vectors_1G); - ms::engine::VectorSourcePtr source_1G = std::make_shared(n_1G, vectors_1G.data()); + milvus::engine::VectorSourcePtr source_1G = std::make_shared(n_1G, vectors_1G.data()); vector_ids.clear(); status = mem_table.Add(source_1G, vector_ids); @@ -209,7 +210,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { vector_ids = source_1G->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_1G); - int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / ms::engine::MAX_TABLE_FILE_MEM); + int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / milvus::engine::MAX_TABLE_FILE_MEM); ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount); status = mem_table.Serialize(); @@ -217,10 +218,10 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { } TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -230,7 +231,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { std::vector xb; BuildVectors(nb, xb); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; for (int64_t i = 0; i < nb; i++) { vector_ids.push_back(i); } @@ -256,9 +257,9 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { } int topk = 10, nprobe = 10; - for (auto &pair : search_vectors) { - auto &search = pair.second; - ms::engine::QueryResults results; + for (auto& pair : search_vectors) { + auto& search = pair.second; + milvus::engine::QueryResults results; stat = db_->Query(GetTableName(), topk, 1, nprobe, search.data(), results); ASSERT_EQ(results[0][0].first, pair.first); ASSERT_LT(results[0][0].second, 1e-4); @@ -266,10 +267,10 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { } TEST_F(MemManagerTest2, INSERT_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -282,7 +283,7 @@ TEST_F(MemManagerTest2, INSERT_TEST) { int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_TRUE(stat.ok()); } @@ -292,17 +293,17 @@ TEST_F(MemManagerTest2, INSERT_TEST) { } TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - ms::engine::IDNumbers vector_ids; - ms::engine::IDNumbers target_ids; + milvus::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers target_ids; int64_t nb = 40960; std::vector xb; @@ -313,7 +314,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { BuildVectors(qb, qxb); std::thread search([&]() { - ms::engine::QueryResults results; + milvus::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -329,7 +330,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { START_TIMER; stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count / ms::engine::M << " M"; + ss << "Search " << j << " With Size " << count / milvus::engine::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -363,16 +364,16 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { } TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { - ms::engine::meta::TableSchema table_info = BuildTableSchema(); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - ms::engine::meta::TableSchema table_info_get; + milvus::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - ms::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers vector_ids; int64_t nb = 100000; std::vector xb; diff --git a/cpp/unittest/db/test_meta.cpp b/cpp/unittest/db/test_meta.cpp index d88c087aa4..e2a9c98089 100644 --- a/cpp/unittest/db/test_meta.cpp +++ b/cpp/unittest/db/test_meta.cpp @@ -26,16 +26,10 @@ #include #include -namespace { - -namespace ms = milvus; - -} // namespace - TEST_F(MetaTest, TABLE_TEST) { auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); @@ -53,7 +47,7 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); + ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); @@ -63,16 +57,16 @@ TEST_F(MetaTest, TABLE_TEST) { TEST_F(MetaTest, TABLE_FILE_TEST) { auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; table.dimension_ = 256; auto status = impl_->CreateTable(table); - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type_, milvus::engine::meta::TableFileSchema::NEW); uint64_t cnt = 0; status = impl_->Count(table_id, cnt); @@ -81,30 +75,30 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { auto file_id = table_file.file_id_; - auto new_file_type = ms::engine::meta::TableFileSchema::INDEX; + auto new_file_type = milvus::engine::meta::TableFileSchema::INDEX; table_file.file_type_ = new_file_type; status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, new_file_type); - ms::engine::meta::DatesT dates; - dates.push_back(ms::engine::utils::GetDate()); + milvus::engine::meta::DatesT dates; + dates.push_back(milvus::engine::utils::GetDate()); status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); dates.clear(); for (auto i = 2; i < 10; ++i) { - dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i)); + dates.push_back(milvus::engine::utils::GetDateWithDelta(-1 * i)); } status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - table_file.date_ = ms::engine::utils::GetDateWithDelta(-2); + table_file.date_ = milvus::engine::utils::GetDateWithDelta(-2); status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2)); - ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date_, milvus::engine::utils::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == milvus::engine::meta::TableFileSchema::TO_DELETE); dates.clear(); dates.push_back(table_file.date_); @@ -112,7 +106,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); std::vector ids = {table_file.id_}; - ms::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), 0UL); @@ -120,34 +114,34 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - ms::engine::DBMetaOptions options; + milvus::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test"; unsigned int seed = 1; int days_num = rand_r(&seed) % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str()); + options.archive_conf_ = milvus::engine::ArchiveConf("delete", ss.str()); - ms::engine::meta::SqliteMetaImpl impl(options); + milvus::engine::meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - ms::engine::meta::TableFilesSchema files; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 100; - int64_t ts = ms::engine::utils::GetMicroSecTimeStamp(); + int64_t ts = milvus::engine::utils::GetMicroSecTimeStamp(); std::vector days; std::vector ids; for (auto i = 0; i < cnt; ++i) { status = impl.CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; int day = rand_r(&seed) % (days_num * 2); - table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000; + table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000; status = impl.UpdateTableFile(table_file); files.push_back(table_file); days.push_back(day); @@ -157,13 +151,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { impl.Archive(); int i = 0; - ms::engine::meta::TableFilesSchema files_get; + milvus::engine::meta::TableFilesSchema files_get; status = impl.GetTableFiles(table_file.table_id_, ids, files_get); ASSERT_TRUE(status.ok()); for (auto &file : files_get) { if (days[i] < days_num) { - ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW); } i++; } @@ -172,19 +166,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { } TEST_F(MetaTest, ARCHIVE_TEST_DISK) { - ms::engine::DBMetaOptions options; + milvus::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test"; - options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11"); + options.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:11"); - ms::engine::meta::SqliteMetaImpl impl(options); + milvus::engine::meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_group"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - ms::engine::meta::TableFilesSchema files; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 10; @@ -192,8 +186,8 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { std::vector ids; for (auto i = 0; i < cnt; ++i) { status = impl.CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; - table_file.file_size_ = each_size * ms::engine::G; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; + table_file.file_size_ = each_size * milvus::engine::G; status = impl.UpdateTableFile(table_file); files.push_back(table_file); ids.push_back(table_file.id_); @@ -202,13 +196,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { impl.Archive(); int i = 0; - ms::engine::meta::TableFilesSchema files_get; + milvus::engine::meta::TableFilesSchema files_get; status = impl.GetTableFiles(table_file.table_id_, ids, files_get); ASSERT_TRUE(status.ok()); for (auto &file : files_get) { if (i >= 5) { - ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW); } ++i; } @@ -219,7 +213,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); @@ -231,51 +225,51 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { uint64_t to_index_files_cnt = 6; uint64_t index_files_cnt = 7; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; for (auto i = 0; i < new_merge_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_MERGE; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < new_index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_INDEX; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < backup_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::BACKUP; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < new_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < raw_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::RAW; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < to_index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } @@ -285,28 +279,28 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt); - ms::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - ms::engine::meta::DatePartionedTableFilesSchema dated_files; + milvus::engine::meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - ms::engine::meta::DatesT dates = {table_file.date_}; + milvus::engine::meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); @@ -321,13 +315,13 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_FALSE(status.ok()); file_types = { - ms::engine::meta::TableFileSchema::NEW, - ms::engine::meta::TableFileSchema::NEW_MERGE, - ms::engine::meta::TableFileSchema::NEW_INDEX, - ms::engine::meta::TableFileSchema::TO_INDEX, - ms::engine::meta::TableFileSchema::INDEX, - ms::engine::meta::TableFileSchema::RAW, - ms::engine::meta::TableFileSchema::BACKUP, + milvus::engine::meta::TableFileSchema::NEW, + milvus::engine::meta::TableFileSchema::NEW_MERGE, + milvus::engine::meta::TableFileSchema::NEW_INDEX, + milvus::engine::meta::TableFileSchema::TO_INDEX, + milvus::engine::meta::TableFileSchema::INDEX, + milvus::engine::meta::TableFileSchema::RAW, + milvus::engine::meta::TableFileSchema::BACKUP, }; status = impl_->FilesByType(table.table_id_, file_types, file_ids); ASSERT_TRUE(status.ok()); @@ -340,7 +334,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); status = impl_->CleanUp(); ASSERT_TRUE(status.ok()); @@ -355,11 +349,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { TEST_F(MetaTest, INDEX_TEST) { auto table_id = "index_test"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); - ms::engine::TableIndex index; + milvus::engine::TableIndex index; index.metric_type_ = 2; index.nlist_ = 1234; index.engine_type_ = 3; @@ -370,12 +364,12 @@ TEST_F(MetaTest, INDEX_TEST) { status = impl_->UpdateTableFlag(table_id, flag); ASSERT_TRUE(status.ok()); - ms::engine::meta::TableSchema table_info; + milvus::engine::meta::TableSchema table_info; table_info.table_id_ = table_id; status = impl_->DescribeTable(table_info); ASSERT_EQ(table_info.flag_, flag); - ms::engine::TableIndex index_out; + milvus::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); ASSERT_EQ(index_out.nlist_, index.nlist_); diff --git a/cpp/unittest/db/test_meta_mysql.cpp b/cpp/unittest/db/test_meta_mysql.cpp index a825e8cbdd..a3259e5645 100644 --- a/cpp/unittest/db/test_meta_mysql.cpp +++ b/cpp/unittest/db/test_meta_mysql.cpp @@ -27,17 +27,10 @@ #include #include -namespace { - -namespace ms = milvus; - -} - - TEST_F(MySqlMetaTest, TABLE_TEST) { auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); @@ -55,7 +48,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); + ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); @@ -68,19 +61,19 @@ TEST_F(MySqlMetaTest, TABLE_TEST) { TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; table.dimension_ = 256; auto status = impl_->CreateTable(table); - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type_, milvus::engine::meta::TableFileSchema::NEW); - ms::engine::meta::DatesT dates; - dates.push_back(ms::engine::utils::GetDate()); + milvus::engine::meta::DatesT dates; + dates.push_back(milvus::engine::utils::GetDate()); status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); @@ -91,7 +84,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { auto file_id = table_file.file_id_; - auto new_file_type = ms::engine::meta::TableFileSchema::INDEX; + auto new_file_type = milvus::engine::meta::TableFileSchema::INDEX; table_file.file_type_ = new_file_type; status = impl_->UpdateTableFile(table_file); @@ -100,16 +93,16 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { dates.clear(); for (auto i = 2; i < 10; ++i) { - dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i)); + dates.push_back(milvus::engine::utils::GetDateWithDelta(-1 * i)); } status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - table_file.date_ = ms::engine::utils::GetDateWithDelta(-2); + table_file.date_ = milvus::engine::utils::GetDateWithDelta(-2); status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2)); - ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date_, milvus::engine::utils::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == milvus::engine::meta::TableFileSchema::TO_DELETE); dates.clear(); dates.push_back(table_file.date_); @@ -117,42 +110,42 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); std::vector ids = {table_file.id_}; - ms::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_EQ(files.size(), 0UL); } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - ms::engine::DBMetaOptions options = GetOptions().meta_; + milvus::engine::DBMetaOptions options = GetOptions().meta_; unsigned int seed = 1; int days_num = rand_r(&seed) % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str()); - int mode = ms::engine::DBOptions::MODE::SINGLE; - ms::engine::meta::MySQLMetaImpl impl(options, mode); + options.archive_conf_ = milvus::engine::ArchiveConf("delete", ss.str()); + int mode = milvus::engine::DBOptions::MODE::SINGLE; + milvus::engine::meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_table"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - ms::engine::meta::TableFilesSchema files; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 100; - int64_t ts = ms::engine::utils::GetMicroSecTimeStamp(); + int64_t ts = milvus::engine::utils::GetMicroSecTimeStamp(); std::vector days; std::vector ids; for (auto i = 0; i < cnt; ++i) { status = impl.CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; int day = rand_r(&seed) % (days_num * 2); - table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000; + table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000; status = impl.UpdateTableFile(table_file); files.push_back(table_file); days.push_back(day); @@ -162,19 +155,19 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { impl.Archive(); int i = 0; - ms::engine::meta::TableFilesSchema files_get; + milvus::engine::meta::TableFilesSchema files_get; status = impl.GetTableFiles(table_file.table_id_, ids, files_get); ASSERT_TRUE(status.ok()); for (auto &file : files_get) { if (days[i] < days_num) { - ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW); } i++; } std::vector file_types = { - (int) ms::engine::meta::TableFileSchema::NEW, + (int) milvus::engine::meta::TableFileSchema::NEW, }; std::vector file_ids; status = impl.FilesByType(table_id, file_types, file_ids); @@ -188,23 +181,23 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { - ms::engine::DBMetaOptions options = GetOptions().meta_; + milvus::engine::DBMetaOptions options = GetOptions().meta_; - options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11"); - int mode = ms::engine::DBOptions::MODE::SINGLE; - auto impl = ms::engine::meta::MySQLMetaImpl(options, mode); + options.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:11"); + int mode = milvus::engine::DBOptions::MODE::SINGLE; + auto impl = milvus::engine::meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - ms::engine::meta::TableSchema table_schema; + milvus::engine::meta::TableSchema table_schema; table_schema.table_id_ = ""; status = impl.CreateTable(table_schema); - ms::engine::meta::TableFilesSchema files; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 10; @@ -212,8 +205,8 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { std::vector ids; for (auto i = 0; i < cnt; ++i) { status = impl.CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; - table_file.file_size_ = each_size * ms::engine::G; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; + table_file.file_size_ = each_size * milvus::engine::G; status = impl.UpdateTableFile(table_file); files.push_back(table_file); ids.push_back(table_file.id_); @@ -222,13 +215,13 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { impl.Archive(); int i = 0; - ms::engine::meta::TableFilesSchema files_get; + milvus::engine::meta::TableFilesSchema files_get; status = impl.GetTableFiles(table_file.table_id_, ids, files_get); ASSERT_TRUE(status.ok()); for (auto &file : files_get) { if (i >= 5) { - ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW); } ++i; } @@ -240,7 +233,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); @@ -252,51 +245,51 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { uint64_t to_index_files_cnt = 6; uint64_t index_files_cnt = 7; - ms::engine::meta::TableFileSchema table_file; + milvus::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; for (auto i = 0; i < new_merge_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_MERGE; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < new_index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_INDEX; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < backup_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::BACKUP; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < new_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < raw_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::RAW; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < to_index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } for (auto i = 0; i < index_files_cnt; ++i) { status = impl_->CreateTableFile(table_file); - table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX; + table_file.file_type_ = milvus::engine::meta::TableFileSchema::INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } @@ -306,28 +299,28 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt); - ms::engine::meta::TableFilesSchema files; + milvus::engine::meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - ms::engine::meta::DatePartionedTableFilesSchema dated_files; + milvus::engine::meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - ms::engine::meta::DatesT dates = {table_file.date_}; + milvus::engine::meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt); @@ -342,13 +335,13 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { ASSERT_FALSE(status.ok()); file_types = { - ms::engine::meta::TableFileSchema::NEW, - ms::engine::meta::TableFileSchema::NEW_MERGE, - ms::engine::meta::TableFileSchema::NEW_INDEX, - ms::engine::meta::TableFileSchema::TO_INDEX, - ms::engine::meta::TableFileSchema::INDEX, - ms::engine::meta::TableFileSchema::RAW, - ms::engine::meta::TableFileSchema::BACKUP, + milvus::engine::meta::TableFileSchema::NEW, + milvus::engine::meta::TableFileSchema::NEW_MERGE, + milvus::engine::meta::TableFileSchema::NEW_INDEX, + milvus::engine::meta::TableFileSchema::TO_INDEX, + milvus::engine::meta::TableFileSchema::INDEX, + milvus::engine::meta::TableFileSchema::RAW, + milvus::engine::meta::TableFileSchema::BACKUP, }; status = impl_->FilesByType(table.table_id_, file_types, file_ids); ASSERT_TRUE(status.ok()); @@ -370,11 +363,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { TEST_F(MySqlMetaTest, INDEX_TEST) { auto table_id = "index_test"; - ms::engine::meta::TableSchema table; + milvus::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); - ms::engine::TableIndex index; + milvus::engine::TableIndex index; index.metric_type_ = 2; index.nlist_ = 1234; index.engine_type_ = 3; @@ -385,12 +378,12 @@ TEST_F(MySqlMetaTest, INDEX_TEST) { status = impl_->UpdateTableFlag(table_id, flag); ASSERT_TRUE(status.ok()); - ms::engine::meta::TableSchema table_info; + milvus::engine::meta::TableSchema table_info; table_info.table_id_ = table_id; status = impl_->DescribeTable(table_info); ASSERT_EQ(table_info.flag_, flag); - ms::engine::TableIndex index_out; + milvus::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); ASSERT_EQ(index_out.nlist_, index.nlist_); diff --git a/cpp/unittest/db/test_misc.cpp b/cpp/unittest/db/test_misc.cpp index 18cca45b4b..f63afc1b27 100644 --- a/cpp/unittest/db/test_misc.cpp +++ b/cpp/unittest/db/test_misc.cpp @@ -27,47 +27,41 @@ #include #include -namespace { - -namespace ms = milvus; - -} // namespace - TEST(DBMiscTest, EXCEPTION_TEST) { - ms::Exception ex1(100, "error"); + milvus::Exception ex1(100, "error"); std::string what = ex1.what(); ASSERT_EQ(what, "error"); ASSERT_EQ(ex1.code(), 100); - ms::InvalidArgumentException ex2; - ASSERT_EQ(ex2.code(), ms::SERVER_INVALID_ARGUMENT); + milvus::InvalidArgumentException ex2; + ASSERT_EQ(ex2.code(), milvus::SERVER_INVALID_ARGUMENT); } TEST(DBMiscTest, OPTIONS_TEST) { try { - ms::engine::ArchiveConf archive("$$##"); + milvus::engine::ArchiveConf archive("$$##"); } catch (std::exception &ex) { ASSERT_TRUE(true); } { - ms::engine::ArchiveConf archive("delete", "no"); + milvus::engine::ArchiveConf archive("delete", "no"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - ms::engine::ArchiveConf archive("delete", "1:2"); + milvus::engine::ArchiveConf archive("delete", "1:2"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - ms::engine::ArchiveConf archive("delete", "1:2:3"); + milvus::engine::ArchiveConf archive("delete", "1:2:3"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - ms::engine::ArchiveConf archive("delete"); - ms::engine::ArchiveConf::CriteriaT criterial = { + milvus::engine::ArchiveConf archive("delete"); + milvus::engine::ArchiveConf::CriteriaT criterial = { {"disk", 1024}, {"days", 100} }; @@ -80,25 +74,25 @@ TEST(DBMiscTest, OPTIONS_TEST) { } TEST(DBMiscTest, META_TEST) { - ms::engine::DBMetaOptions options; + milvus::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test"; - ms::engine::meta::SqliteMetaImpl impl(options); + milvus::engine::meta::SqliteMetaImpl impl(options); time_t tt; time(&tt); int delta = 10; - ms::engine::meta::DateT dt = ms::engine::utils::GetDate(tt, delta); + milvus::engine::meta::DateT dt = milvus::engine::utils::GetDate(tt, delta); ASSERT_GT(dt, 0); } TEST(DBMiscTest, UTILS_TEST) { - ms::engine::DBMetaOptions options; + milvus::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test/main"; options.slave_paths_.push_back("/tmp/milvus_test/slave_1"); options.slave_paths_.push_back("/tmp/milvus_test/slave_2"); const std::string TABLE_NAME = "test_tbl"; - auto status = ms::engine::utils::CreateTablePath(options, TABLE_NAME); + auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME); ASSERT_TRUE(status.ok()); ASSERT_TRUE(boost::filesystem::exists(options.path_)); for (auto &path : options.slave_paths_) { @@ -113,18 +107,18 @@ TEST(DBMiscTest, UTILS_TEST) { // status = engine::utils::CreateTablePath(options, TABLE_NAME); // ASSERT_FALSE(status.ok()); - ms::engine::meta::TableFileSchema file; + milvus::engine::meta::TableFileSchema file; file.id_ = 50; file.table_id_ = TABLE_NAME; file.file_type_ = 3; file.date_ = 155000; - status = ms::engine::utils::GetTableFilePath(options, file); + status = milvus::engine::utils::GetTableFilePath(options, file); ASSERT_FALSE(status.ok()); ASSERT_TRUE(file.location_.empty()); - status = ms::engine::utils::DeleteTablePath(options, TABLE_NAME); + status = milvus::engine::utils::DeleteTablePath(options, TABLE_NAME); ASSERT_TRUE(status.ok()); - status = ms::engine::utils::DeleteTableFilePath(options, file); + status = milvus::engine::utils::DeleteTableFilePath(options, file); ASSERT_TRUE(status.ok()); } diff --git a/cpp/unittest/db/test_search.cpp b/cpp/unittest/db/test_search.cpp index 348463357e..e17e06ac16 100644 --- a/cpp/unittest/db/test_search.cpp +++ b/cpp/unittest/db/test_search.cpp @@ -30,8 +30,8 @@ void BuildResult(uint64_t nq, uint64_t topk, bool ascending, - std::vector &output_ids, - std::vector &output_distence) { + std::vector& output_ids, + std::vector& output_distence) { output_ids.clear(); output_ids.resize(nq * topk); output_distence.clear(); @@ -39,20 +39,21 @@ BuildResult(uint64_t nq, for (uint64_t i = 0; i < nq; i++) { for (uint64_t j = 0; j < topk; j++) { - output_ids[i * topk + j] = (int64_t) (drand48() * 100000); + output_ids[i * topk + j] = (int64_t)(drand48() * 100000); output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48()); } } } -void CheckTopkResult(const std::vector &input_ids_1, - const std::vector &input_distance_1, - const std::vector &input_ids_2, - const std::vector &input_distance_2, - uint64_t nq, - uint64_t topk, - bool ascending, - const ms::ResultSet& result) { +void +CheckTopkResult(const std::vector& input_ids_1, + const std::vector& input_distance_1, + const std::vector& input_ids_2, + const std::vector& input_distance_2, + uint64_t nq, + uint64_t topk, + bool ascending, + const milvus::scheduler::ResultSet& result) { ASSERT_EQ(result.size(), nq); ASSERT_EQ(input_ids_1.size(), input_distance_1.size()); ASSERT_EQ(input_ids_2.size(), input_distance_2.size()); @@ -61,15 +62,18 @@ void CheckTopkResult(const std::vector &input_ids_1, uint64_t input_k2 = input_ids_2.size() / nq; for (int64_t i = 0; i < nq; i++) { - std::vector src_vec(input_distance_1.begin()+i*input_k1, input_distance_1.begin()+(i+1)*input_k1); - src_vec.insert(src_vec.end(), input_distance_2.begin()+i*input_k2, input_distance_2.begin()+(i+1)*input_k2); + std::vector + src_vec(input_distance_1.begin() + i * input_k1, input_distance_1.begin() + (i + 1) * input_k1); + src_vec.insert(src_vec.end(), + input_distance_2.begin() + i * input_k2, + input_distance_2.begin() + (i + 1) * input_k2); if (ascending) { std::sort(src_vec.begin(), src_vec.end()); } else { std::sort(src_vec.begin(), src_vec.end(), std::greater()); } - uint64_t n = std::min(topk, input_k1+input_k2); + uint64_t n = std::min(topk, input_k1 + input_k2); for (uint64_t j = 0; j < n; j++) { if (src_vec[j] != result[i][j].second) { std::cout << src_vec[j] << " " << result[i][j].second << std::endl; @@ -87,19 +91,19 @@ TEST(DBSearchTest, TOPK_TEST) { bool ascending; std::vector ids1, ids2; std::vector dist1, dist2; - ms::ResultSet result; + milvus::scheduler::ResultSet result; milvus::Status status; /* test1, id1/dist1 valid, id2/dist2 empty */ ascending = true; BuildResult(NQ, TOP_K, ascending, ids1, dist1); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); /* test2, id1/dist1 valid, id2/dist2 valid */ BuildResult(NQ, TOP_K, ascending, ids2, dist2); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); @@ -107,10 +111,10 @@ TEST(DBSearchTest, TOPK_TEST) { ids1.clear(); dist1.clear(); result.clear(); - BuildResult(NQ, TOP_K/2, ascending, ids1, dist1); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result); + BuildResult(NQ, TOP_K / 2, ascending, ids1, dist1); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); @@ -118,10 +122,10 @@ TEST(DBSearchTest, TOPK_TEST) { ids2.clear(); dist2.clear(); result.clear(); - BuildResult(NQ, TOP_K/3, ascending, ids2, dist2); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result); + BuildResult(NQ, TOP_K / 3, ascending, ids2, dist2); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K / 3, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); @@ -135,13 +139,13 @@ TEST(DBSearchTest, TOPK_TEST) { /* test1, id1/dist1 valid, id2/dist2 empty */ BuildResult(NQ, TOP_K, ascending, ids1, dist1); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); /* test2, id1/dist1 valid, id2/dist2 valid */ BuildResult(NQ, TOP_K, ascending, ids2, dist2); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); @@ -149,10 +153,10 @@ TEST(DBSearchTest, TOPK_TEST) { ids1.clear(); dist1.clear(); result.clear(); - BuildResult(NQ, TOP_K/2, ascending, ids1, dist1); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result); + BuildResult(NQ, TOP_K / 2, ascending, ids1, dist1); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); @@ -160,10 +164,10 @@ TEST(DBSearchTest, TOPK_TEST) { ids2.clear(); dist2.clear(); result.clear(); - BuildResult(NQ, TOP_K/3, ascending, ids2, dist2); - status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result); + BuildResult(NQ, TOP_K / 3, ascending, ids2, dist2); + status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); - status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result); + status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K / 3, NQ, TOP_K, ascending, result); ASSERT_TRUE(status.ok()); CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result); } @@ -175,7 +179,7 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) { bool ascending = true; std::vector input_ids; std::vector input_distance; - ms::ResultSet final_result; + milvus::scheduler::ResultSet final_result; milvus::Status status; double span, reduce_cost = 0.0; @@ -187,12 +191,18 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) { rc.RecordSection("do search for context: " + std::to_string(i)); // pick up topk result - status = ms::XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result); + status = milvus::scheduler::XSearchTask::TopkResult(input_ids, + input_distance, + top_k, + nq, + top_k, + ascending, + final_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(final_result.size(), nq); span = rc.RecordSection("reduce topk for context: " + std::to_string(i)); reduce_cost += span; } - std::cout << "total reduce time: " << reduce_cost/1000 << " ms" << std::endl; + std::cout << "total reduce time: " << reduce_cost / 1000 << " ms" << std::endl; } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index c5874be694..dfbffc66bc 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -33,8 +33,6 @@ INITIALIZE_EASYLOGGINGPP namespace { -namespace ms = milvus; - class DBTestEnvironment : public ::testing::Environment { public: explicit DBTestEnvironment(const std::string& uri) @@ -83,9 +81,9 @@ BaseTest::TearDown() { knowhere::FaissGpuResourceMgr::GetInstance().Free(); } -ms::engine::DBOptions +milvus::engine::DBOptions BaseTest::GetOptions() { - auto options = ms::engine::DBFactory::BuildOption(); + auto options = milvus::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = "sqlite://:@:/"; return options; @@ -96,23 +94,23 @@ void DBTest::SetUp() { BaseTest::SetUp(); - auto res_mgr = ms::scheduler::ResMgrInst::GetInstance(); + auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = ms::scheduler::Connection("IO", 500.0); - auto PCIE = ms::scheduler::Connection("IO", 11000.0); + auto default_conn = milvus::scheduler::Connection("IO", 500.0); + auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - ms::scheduler::SchedInst::GetInstance()->Start(); + milvus::scheduler::SchedInst::GetInstance()->Start(); - ms::scheduler::JobMgrInst::GetInstance()->Start(); + milvus::scheduler::JobMgrInst::GetInstance()->Start(); auto options = GetOptions(); - db_ = ms::engine::DBFactory::Build(options); + db_ = milvus::engine::DBFactory::Build(options); } void @@ -120,10 +118,10 @@ DBTest::TearDown() { db_->Stop(); db_->DropAll(); - ms::scheduler::JobMgrInst::GetInstance()->Stop(); - ms::scheduler::SchedInst::GetInstance()->Stop(); - ms::scheduler::ResMgrInst::GetInstance()->Stop(); - ms::scheduler::ResMgrInst::GetInstance()->Clear(); + milvus::scheduler::JobMgrInst::GetInstance()->Stop(); + milvus::scheduler::SchedInst::GetInstance()->Stop(); + milvus::scheduler::ResMgrInst::GetInstance()->Stop(); + milvus::scheduler::ResMgrInst::GetInstance()->Clear(); BaseTest::TearDown(); @@ -132,11 +130,11 @@ DBTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ms::engine::DBOptions +milvus::engine::DBOptions DBTest2::GetOptions() { - auto options = ms::engine::DBFactory::BuildOption(); + auto options = milvus::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; - options.meta_.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:1"); + options.meta_.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:1"); options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } @@ -147,7 +145,7 @@ MetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta_); + impl_ = std::make_shared(options.meta_); } void @@ -161,9 +159,9 @@ MetaTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ms::engine::DBOptions +milvus::engine::DBOptions MySqlDBTest::GetOptions() { - auto options = ms::engine::DBFactory::BuildOption(); + auto options = milvus::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = test_env->getURI(); @@ -176,7 +174,7 @@ MySqlMetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta_, options.mode_); + impl_ = std::make_shared(options.meta_, options.mode_); } void @@ -189,9 +187,9 @@ MySqlMetaTest::TearDown() { boost::filesystem::remove_all(options.meta_.path_); } -ms::engine::DBOptions +milvus::engine::DBOptions MySqlMetaTest::GetOptions() { - auto options = ms::engine::DBFactory::BuildOption(); + auto options = milvus::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = test_env->getURI(); diff --git a/cpp/unittest/metrics/test_metricbase.cpp b/cpp/unittest/metrics/test_metricbase.cpp index 4edb3b8151..c2d43642ab 100644 --- a/cpp/unittest/metrics/test_metricbase.cpp +++ b/cpp/unittest/metrics/test_metricbase.cpp @@ -21,16 +21,10 @@ #include #include -namespace { - -namespace ms = milvus; - -} // namespace - TEST(MetricbaseTest, METRICBASE_TEST) { - ms::server::MetricsBase instance = ms::server::MetricsBase::GetInstance(); + milvus::server::MetricsBase instance = milvus::server::MetricsBase::GetInstance(); instance.Init(); - ms::server::SystemInfo::GetInstance().Init(); + milvus::server::SystemInfo::GetInstance().Init(); instance.AddVectorsSuccessTotalIncrement(); instance.AddVectorsFailTotalIncrement(); instance.AddVectorsDurationHistogramOberve(1.0); diff --git a/cpp/unittest/metrics/test_metrics.cpp b/cpp/unittest/metrics/test_metrics.cpp index ba0b51af00..c0d1044bb4 100644 --- a/cpp/unittest/metrics/test_metrics.cpp +++ b/cpp/unittest/metrics/test_metrics.cpp @@ -29,22 +29,16 @@ #include "db/DB.h" #include "db/meta/SqliteMetaImpl.h" -namespace { - -namespace ms = milvus; - -} // namespace - TEST_F(MetricTest, METRIC_TEST) { - ms::server::Config::GetInstance().SetMetricConfigCollector("zabbix"); - ms::server::Metrics::GetInstance(); - ms::server::Config::GetInstance().SetMetricConfigCollector("prometheus"); - ms::server::Metrics::GetInstance(); + milvus::server::Config::GetInstance().SetMetricConfigCollector("zabbix"); + milvus::server::Metrics::GetInstance(); + milvus::server::Config::GetInstance().SetMetricConfigCollector("prometheus"); + milvus::server::Metrics::GetInstance(); - ms::server::SystemInfo::GetInstance().Init(); + milvus::server::SystemInfo::GetInstance().Init(); // server::Metrics::GetInstance().Init(); // server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr()); - ms::server::Metrics::GetInstance().Init(); + milvus::server::Metrics::GetInstance().Init(); // server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr()); milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024); @@ -53,17 +47,17 @@ TEST_F(MetricTest, METRIC_TEST) { static const char *group_name = "test_group"; static const int group_dim = 256; - ms::engine::meta::TableSchema group_info; + milvus::engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; auto stat = db_->CreateTable(group_info); - ms::engine::meta::TableSchema group_info_get; + milvus::engine::meta::TableSchema group_info_get; group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); - ms::engine::IDNumbers vector_ids; - ms::engine::IDNumbers target_ids; + milvus::engine::IDNumbers vector_ids; + milvus::engine::IDNumbers target_ids; int d = 256; int nb = 50; @@ -81,7 +75,7 @@ TEST_F(MetricTest, METRIC_TEST) { } std::thread search([&]() { - ms::engine::QueryResults results; + milvus::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -132,32 +126,32 @@ TEST_F(MetricTest, METRIC_TEST) { } TEST_F(MetricTest, COLLECTOR_METRICS_TEST) { - auto status = ms::Status::OK(); - ms::server::CollectInsertMetrics insert_metrics0(0, status); - status = ms::Status(ms::DB_ERROR, "error"); - ms::server::CollectInsertMetrics insert_metrics1(0, status); + auto status = milvus::Status::OK(); + milvus::server::CollectInsertMetrics insert_metrics0(0, status); + status = milvus::Status(milvus::DB_ERROR, "error"); + milvus::server::CollectInsertMetrics insert_metrics1(0, status); - ms::server::CollectQueryMetrics query_metrics(10); + milvus::server::CollectQueryMetrics query_metrics(10); - ms::server::CollectMergeFilesMetrics merge_metrics(); + milvus::server::CollectMergeFilesMetrics merge_metrics(); - ms::server::CollectBuildIndexMetrics build_index_metrics(); + milvus::server::CollectBuildIndexMetrics build_index_metrics(); - ms::server::CollectExecutionEngineMetrics execution_metrics(10); + milvus::server::CollectExecutionEngineMetrics execution_metrics(10); - ms::server::CollectSerializeMetrics serialize_metrics(10); + milvus::server::CollectSerializeMetrics serialize_metrics(10); - ms::server::CollectAddMetrics add_metrics(10, 128); + milvus::server::CollectAddMetrics add_metrics(10, 128); - ms::server::CollectDurationMetrics duration_metrics_raw(ms::engine::meta::TableFileSchema::RAW); - ms::server::CollectDurationMetrics duration_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX); - ms::server::CollectDurationMetrics duration_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE); + milvus::server::CollectDurationMetrics duration_metrics_raw(milvus::engine::meta::TableFileSchema::RAW); + milvus::server::CollectDurationMetrics duration_metrics_index(milvus::engine::meta::TableFileSchema::TO_INDEX); + milvus::server::CollectDurationMetrics duration_metrics_delete(milvus::engine::meta::TableFileSchema::TO_DELETE); - ms::server::CollectSearchTaskMetrics search_metrics_raw(ms::engine::meta::TableFileSchema::RAW); - ms::server::CollectSearchTaskMetrics search_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX); - ms::server::CollectSearchTaskMetrics search_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE); + milvus::server::CollectSearchTaskMetrics search_metrics_raw(milvus::engine::meta::TableFileSchema::RAW); + milvus::server::CollectSearchTaskMetrics search_metrics_index(milvus::engine::meta::TableFileSchema::TO_INDEX); + milvus::server::CollectSearchTaskMetrics search_metrics_delete(milvus::engine::meta::TableFileSchema::TO_DELETE); - ms::server::MetricCollector metric_collector(); + milvus::server::MetricCollector metric_collector(); } diff --git a/cpp/unittest/metrics/test_prometheus.cpp b/cpp/unittest/metrics/test_prometheus.cpp index 14982d058d..e90dbd56bc 100644 --- a/cpp/unittest/metrics/test_prometheus.cpp +++ b/cpp/unittest/metrics/test_prometheus.cpp @@ -22,19 +22,13 @@ #include #include -namespace { - -namespace ms = milvus; - -} // namespace - TEST(PrometheusTest, PROMETHEUS_TEST) { - ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("on"); + milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("on"); - ms::server::PrometheusMetrics instance = ms::server::PrometheusMetrics::GetInstance(); + milvus::server::PrometheusMetrics instance = milvus::server::PrometheusMetrics::GetInstance(); instance.Init(); instance.SetStartup(true); - ms::server::SystemInfo::GetInstance().Init(); + milvus::server::SystemInfo::GetInstance().Init(); instance.AddVectorsSuccessTotalIncrement(); instance.AddVectorsFailTotalIncrement(); instance.AddVectorsDurationHistogramOberve(1.0); @@ -80,7 +74,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST) { instance.GPUTemperature(); instance.CPUTemperature(); - ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("off"); + milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("off"); instance.Init(); instance.CPUCoreUsagePercentSet(); instance.GPUTemperature(); diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index e345923b7b..72a172c4ee 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -28,8 +28,6 @@ INITIALIZE_EASYLOGGINGPP namespace { -namespace ms = milvus; - class DBTestEnvironment : public ::testing::Environment { public: explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {} @@ -58,8 +56,8 @@ void MetricTest::InitLog() { el::Loggers::reconfigureLogger("default", defaultConf); } -ms::engine::DBOptions MetricTest::GetOptions() { - auto options = ms::engine::DBFactory::BuildOption(); +milvus::engine::DBOptions MetricTest::GetOptions() { + auto options = milvus::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = "sqlite://:@:/"; return options; @@ -69,7 +67,7 @@ void MetricTest::SetUp() { boost::filesystem::remove_all("/tmp/milvus_test"); InitLog(); auto options = GetOptions(); - db_ = ms::engine::DBFactory::Build(options); + db_ = milvus::engine::DBFactory::Build(options); } void MetricTest::TearDown() { diff --git a/cpp/unittest/scheduler/test_tasktable.cpp b/cpp/unittest/scheduler/test_tasktable.cpp index 271826614d..e717e40285 100644 --- a/cpp/unittest/scheduler/test_tasktable.cpp +++ b/cpp/unittest/scheduler/test_tasktable.cpp @@ -21,53 +21,47 @@ #include "scheduler/tasklabel/DefaultLabel.h" #include -namespace { - -namespace ms = milvus::scheduler; - -} // namespace - /************ TaskTableBaseTest ************/ class TaskTableItemTest : public ::testing::Test { protected: void SetUp() override { - std::vector states{ - ms::TaskTableItemState::INVALID, - ms::TaskTableItemState::START, - ms::TaskTableItemState::LOADING, - ms::TaskTableItemState::LOADED, - ms::TaskTableItemState::EXECUTING, - ms::TaskTableItemState::EXECUTED, - ms::TaskTableItemState::MOVING, - ms::TaskTableItemState::MOVED}; + std::vector states{ + milvus::scheduler::TaskTableItemState::INVALID, + milvus::scheduler::TaskTableItemState::START, + milvus::scheduler::TaskTableItemState::LOADING, + milvus::scheduler::TaskTableItemState::LOADED, + milvus::scheduler::TaskTableItemState::EXECUTING, + milvus::scheduler::TaskTableItemState::EXECUTED, + milvus::scheduler::TaskTableItemState::MOVING, + milvus::scheduler::TaskTableItemState::MOVED}; for (auto &state : states) { - auto item = std::make_shared(); + auto item = std::make_shared(); item->state = state; items_.emplace_back(item); } } - ms::TaskTableItem default_; - std::vector items_; + milvus::scheduler::TaskTableItem default_; + std::vector items_; }; TEST_F(TaskTableItemTest, CONSTRUCT) { ASSERT_EQ(default_.id, 0); ASSERT_EQ(default_.task, nullptr); - ASSERT_EQ(default_.state, ms::TaskTableItemState::INVALID); + ASSERT_EQ(default_.state, milvus::scheduler::TaskTableItemState::INVALID); } TEST_F(TaskTableItemTest, DESTRUCT) { - auto p_item = new ms::TaskTableItem(); + auto p_item = new milvus::scheduler::TaskTableItem(); delete p_item; } TEST_F(TaskTableItemTest, IS_FINISH) { for (auto &item : items_) { - if (item->state == ms::TaskTableItemState::EXECUTED - || item->state == ms::TaskTableItemState::MOVED) { + if (item->state == milvus::scheduler::TaskTableItemState::EXECUTED + || item->state == milvus::scheduler::TaskTableItemState::MOVED) { ASSERT_TRUE(item->IsFinish()); } else { ASSERT_FALSE(item->IsFinish()); @@ -85,9 +79,9 @@ TEST_F(TaskTableItemTest, LOAD) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Load(); - if (before_state == ms::TaskTableItemState::START) { + if (before_state == milvus::scheduler::TaskTableItemState::START) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::LOADING); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::LOADING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -99,9 +93,9 @@ TEST_F(TaskTableItemTest, LOADED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Loaded(); - if (before_state == ms::TaskTableItemState::LOADING) { + if (before_state == milvus::scheduler::TaskTableItemState::LOADING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::LOADED); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::LOADED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -113,9 +107,9 @@ TEST_F(TaskTableItemTest, EXECUTE) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Execute(); - if (before_state == ms::TaskTableItemState::LOADED) { + if (before_state == milvus::scheduler::TaskTableItemState::LOADED) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTING); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::EXECUTING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -127,9 +121,9 @@ TEST_F(TaskTableItemTest, EXECUTED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Executed(); - if (before_state == ms::TaskTableItemState::EXECUTING) { + if (before_state == milvus::scheduler::TaskTableItemState::EXECUTING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTED); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::EXECUTED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -141,9 +135,9 @@ TEST_F(TaskTableItemTest, MOVE) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Move(); - if (before_state == ms::TaskTableItemState::LOADED) { + if (before_state == milvus::scheduler::TaskTableItemState::LOADED) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::MOVING); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::MOVING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -155,9 +149,9 @@ TEST_F(TaskTableItemTest, MOVED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Moved(); - if (before_state == ms::TaskTableItemState::MOVING) { + if (before_state == milvus::scheduler::TaskTableItemState::MOVING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, ms::TaskTableItemState::MOVED); + ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::MOVED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -171,17 +165,17 @@ class TaskTableBaseTest : public ::testing::Test { protected: void SetUp() override { - ms::TableFileSchemaPtr dummy = nullptr; + milvus::scheduler::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - auto label = std::make_shared(); - task1_ = std::make_shared(dummy, label); - task2_ = std::make_shared(dummy, label); + auto label = std::make_shared(); + task1_ = std::make_shared(dummy, label); + task2_ = std::make_shared(dummy, label); } - ms::TaskPtr invalid_task_; - ms::TaskPtr task1_; - ms::TaskPtr task2_; - ms::TaskTable empty_table_; + milvus::scheduler::TaskPtr invalid_task_; + milvus::scheduler::TaskPtr task1_; + milvus::scheduler::TaskPtr task2_; + milvus::scheduler::TaskTable empty_table_; }; TEST_F(TaskTableBaseTest, SUBSCRIBER) { @@ -205,14 +199,14 @@ TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) { } TEST_F(TaskTableBaseTest, PUT_BATCH) { - std::vector tasks{task1_, task2_}; + std::vector tasks{task1_, task2_}; empty_table_.Put(tasks); ASSERT_EQ(empty_table_.Get(0)->task, task1_); ASSERT_EQ(empty_table_.Get(1)->task, task2_); } TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) { - std::vector tasks{}; + std::vector tasks{}; empty_table_.Put(tasks); } @@ -238,8 +232,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; auto indexes = empty_table_.PickToLoad(1); ASSERT_EQ(indexes.size(), 1); @@ -251,8 +245,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; auto indexes = empty_table_.PickToLoad(3); ASSERT_EQ(indexes.size(), 3); @@ -266,8 +260,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; // first pick, non-cache auto indexes = empty_table_.PickToLoad(1); @@ -276,7 +270,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) { // second pick, iterate from 2 // invalid state change - empty_table_[1]->state = ms::TaskTableItemState::START; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::START; indexes = empty_table_.PickToLoad(1); ASSERT_EQ(indexes.size(), 1); ASSERT_EQ(indexes[0], 2); @@ -287,9 +281,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; - empty_table_[2]->state = ms::TaskTableItemState::LOADED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; + empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED; auto indexes = empty_table_.PickToExecute(1); ASSERT_EQ(indexes.size(), 1); @@ -301,10 +295,10 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; - empty_table_[2]->state = ms::TaskTableItemState::LOADED; - empty_table_[3]->state = ms::TaskTableItemState::LOADED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; + empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED; + empty_table_[3]->state = milvus::scheduler::TaskTableItemState::LOADED; auto indexes = empty_table_.PickToExecute(3); ASSERT_EQ(indexes.size(), 2); @@ -317,9 +311,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = ms::TaskTableItemState::MOVED; - empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; - empty_table_[2]->state = ms::TaskTableItemState::LOADED; + empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED; + empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED; // first pick, non-cache auto indexes = empty_table_.PickToExecute(1); @@ -328,7 +322,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) { // second pick, iterate from 2 // invalid state change - empty_table_[1]->state = ms::TaskTableItemState::START; + empty_table_[1]->state = milvus::scheduler::TaskTableItemState::START; indexes = empty_table_.PickToExecute(1); ASSERT_EQ(indexes.size(), 1); ASSERT_EQ(indexes[0], 2); @@ -340,28 +334,28 @@ class TaskTableAdvanceTest : public ::testing::Test { protected: void SetUp() override { - ms::TableFileSchemaPtr dummy = nullptr; + milvus::scheduler::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); + auto label = std::make_shared(); + auto task = std::make_shared(dummy, label); table1_.Put(task); } - table1_.Get(0)->state = ms::TaskTableItemState::INVALID; - table1_.Get(1)->state = ms::TaskTableItemState::START; - table1_.Get(2)->state = ms::TaskTableItemState::LOADING; - table1_.Get(3)->state = ms::TaskTableItemState::LOADED; - table1_.Get(4)->state = ms::TaskTableItemState::EXECUTING; - table1_.Get(5)->state = ms::TaskTableItemState::EXECUTED; - table1_.Get(6)->state = ms::TaskTableItemState::MOVING; - table1_.Get(7)->state = ms::TaskTableItemState::MOVED; + table1_.Get(0)->state = milvus::scheduler::TaskTableItemState::INVALID; + table1_.Get(1)->state = milvus::scheduler::TaskTableItemState::START; + table1_.Get(2)->state = milvus::scheduler::TaskTableItemState::LOADING; + table1_.Get(3)->state = milvus::scheduler::TaskTableItemState::LOADED; + table1_.Get(4)->state = milvus::scheduler::TaskTableItemState::EXECUTING; + table1_.Get(5)->state = milvus::scheduler::TaskTableItemState::EXECUTED; + table1_.Get(6)->state = milvus::scheduler::TaskTableItemState::MOVING; + table1_.Get(7)->state = milvus::scheduler::TaskTableItemState::MOVED; } - ms::TaskTable table1_; + milvus::scheduler::TaskTable table1_; }; TEST_F(TaskTableAdvanceTest, LOAD) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -371,8 +365,8 @@ TEST_F(TaskTableAdvanceTest, LOAD) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::START) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADING); + if (before_state[i] == milvus::scheduler::TaskTableItemState::START) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::LOADING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -380,7 +374,7 @@ TEST_F(TaskTableAdvanceTest, LOAD) { } TEST_F(TaskTableAdvanceTest, LOADED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -390,8 +384,8 @@ TEST_F(TaskTableAdvanceTest, LOADED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::LOADING) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADED); + if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADING) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::LOADED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -399,7 +393,7 @@ TEST_F(TaskTableAdvanceTest, LOADED) { } TEST_F(TaskTableAdvanceTest, EXECUTE) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -409,8 +403,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::LOADED) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTING); + if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADED) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::EXECUTING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -418,7 +412,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) { } TEST_F(TaskTableAdvanceTest, EXECUTED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -428,8 +422,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::EXECUTING) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTED); + if (before_state[i] == milvus::scheduler::TaskTableItemState::EXECUTING) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::EXECUTED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -437,7 +431,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) { } TEST_F(TaskTableAdvanceTest, MOVE) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -447,8 +441,8 @@ TEST_F(TaskTableAdvanceTest, MOVE) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::LOADED) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVING); + if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADED) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::MOVING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -456,7 +450,7 @@ TEST_F(TaskTableAdvanceTest, MOVE) { } TEST_F(TaskTableAdvanceTest, MOVED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -466,8 +460,8 @@ TEST_F(TaskTableAdvanceTest, MOVED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == ms::TaskTableItemState::MOVING) { - ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVED); + if (before_state[i] == milvus::scheduler::TaskTableItemState::MOVING) { + ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::MOVED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } diff --git a/cpp/unittest/server/test_cache.cpp b/cpp/unittest/server/test_cache.cpp index 8774851c43..dd55e7aef6 100644 --- a/cpp/unittest/server/test_cache.cpp +++ b/cpp/unittest/server/test_cache.cpp @@ -23,38 +23,36 @@ namespace { -namespace ms = milvus; - -class InvalidCacheMgr : public ms::cache::CacheMgr { +class InvalidCacheMgr : public milvus::cache::CacheMgr { public: InvalidCacheMgr() { } }; -class LessItemCacheMgr : public ms::cache::CacheMgr { +class LessItemCacheMgr : public milvus::cache::CacheMgr { public: LessItemCacheMgr() { - cache_ = std::make_shared>(1UL << 12, 10); + cache_ = std::make_shared>(1UL << 12, 10); } }; -class MockVecIndex : public ms::engine::VecIndex { +class MockVecIndex : public milvus::engine::VecIndex { public: MockVecIndex(int64_t dim, int64_t total) : dimension_(dim), ntotal_(total) { } - virtual ms::Status BuildAll(const int64_t &nb, + virtual milvus::Status BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, - const ms::engine::Config &cfg, + const milvus::engine::Config &cfg, const int64_t &nt = 0, const float *xt = nullptr) { - return ms::Status(); + return milvus::Status(); } - ms::engine::VecIndexPtr Clone() override { + milvus::engine::VecIndexPtr Clone() override { return milvus::engine::VecIndexPtr(); } @@ -62,31 +60,31 @@ class MockVecIndex : public ms::engine::VecIndex { return 0; } - ms::engine::IndexType GetType() override { - return ms::engine::IndexType::INVALID; + milvus::engine::IndexType GetType() override { + return milvus::engine::IndexType::INVALID; } - virtual ms::Status Add(const int64_t &nb, + virtual milvus::Status Add(const int64_t &nb, const float *xb, const int64_t *ids, - const ms::engine::Config &cfg = ms::engine::Config()) { - return ms::Status(); + const milvus::engine::Config &cfg = milvus::engine::Config()) { + return milvus::Status(); } - virtual ms::Status Search(const int64_t &nq, + virtual milvus::Status Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, - const ms::engine::Config &cfg = ms::engine::Config()) { - return ms::Status(); + const milvus::engine::Config &cfg = milvus::engine::Config()) { + return milvus::Status(); } - ms::engine::VecIndexPtr CopyToGpu(const int64_t &device_id, - const ms::engine::Config &cfg) override { + milvus::engine::VecIndexPtr CopyToGpu(const int64_t &device_id, + const milvus::engine::Config &cfg) override { return nullptr; } - ms::engine::VecIndexPtr CopyToCpu(const ms::engine::Config &cfg) override { + milvus::engine::VecIndexPtr CopyToCpu(const milvus::engine::Config &cfg) override { return nullptr; } @@ -103,8 +101,8 @@ class MockVecIndex : public ms::engine::VecIndex { return binset; } - virtual ms::Status Load(const knowhere::BinarySet &index_binary) { - return ms::Status(); + virtual milvus::Status Load(const knowhere::BinarySet &index_binary) { + return milvus::Status(); } public: @@ -115,7 +113,7 @@ class MockVecIndex : public ms::engine::VecIndex { } // namespace TEST(CacheTest, DUMMY_TEST) { - ms::engine::Config cfg; + milvus::engine::Config cfg; MockVecIndex mock_index(256, 1000); mock_index.Dimension(); mock_index.Count(); @@ -133,7 +131,7 @@ TEST(CacheTest, DUMMY_TEST) { } TEST(CacheTest, CPU_CACHE_TEST) { - auto cpu_mgr = ms::cache::CpuCacheMgr::GetInstance(); + auto cpu_mgr = milvus::cache::CpuCacheMgr::GetInstance(); const int64_t gbyte = 1024 * 1024 * 1024; int64_t g_num = 16; @@ -144,8 +142,8 @@ TEST(CacheTest, CPU_CACHE_TEST) { uint64_t item_count = 20; for (uint64_t i = 0; i < item_count; i++) { //each vector is 1k byte, total size less than 1G - ms::engine::VecIndexPtr mock_index = std::make_shared(256, 1000000); - ms::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); + milvus::engine::VecIndexPtr mock_index = std::make_shared(256, 1000000); + milvus::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_LT(cpu_mgr->ItemCount(), g_num); @@ -168,8 +166,8 @@ TEST(CacheTest, CPU_CACHE_TEST) { cpu_mgr->SetCapacity(g_num * gbyte); //each vector is 1k byte, total size less than 6G - ms::engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); - ms::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); + milvus::engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); + milvus::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); cpu_mgr->InsertItem("index_6g", data_obj); ASSERT_TRUE(cpu_mgr->ItemExists("index_6g")); } @@ -178,12 +176,12 @@ TEST(CacheTest, CPU_CACHE_TEST) { } TEST(CacheTest, GPU_CACHE_TEST) { - auto gpu_mgr = ms::cache::GpuCacheMgr::GetInstance(0); + auto gpu_mgr = milvus::cache::GpuCacheMgr::GetInstance(0); for (int i = 0; i < 20; i++) { //each vector is 1k byte - ms::engine::VecIndexPtr mock_index = std::make_shared(256, 1000); - ms::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); + milvus::engine::VecIndexPtr mock_index = std::make_shared(256, 1000); + milvus::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } @@ -195,8 +193,8 @@ TEST(CacheTest, GPU_CACHE_TEST) { for (auto i = 0; i < 3; i++) { // TODO(myh): use gpu index to mock //each vector is 1k byte, total size less than 2G - ms::engine::VecIndexPtr mock_index = std::make_shared(256, 2000000); - ms::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); + milvus::engine::VecIndexPtr mock_index = std::make_shared(256, 2000000); + milvus::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); std::cout << data_obj->Size() << std::endl; gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } @@ -212,7 +210,7 @@ TEST(CacheTest, INVALID_TEST) { ASSERT_FALSE(mgr.ItemExists("test")); ASSERT_EQ(mgr.GetItem("test"), nullptr); - mgr.InsertItem("test", ms::cache::DataObjPtr()); + mgr.InsertItem("test", milvus::cache::DataObjPtr()); mgr.InsertItem("test", nullptr); mgr.EraseItem("test"); mgr.PrintInfo(); @@ -226,8 +224,8 @@ TEST(CacheTest, INVALID_TEST) { LessItemCacheMgr mgr; for (int i = 0; i < 20; i++) { //each vector is 1k byte - ms::engine::VecIndexPtr mock_index = std::make_shared(256, 2); - ms::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); + milvus::engine::VecIndexPtr mock_index = std::make_shared(256, 2); + milvus::cache::DataObjPtr data_obj = std::static_pointer_cast(mock_index); mgr.InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_EQ(mgr.GetItem("index_0"), nullptr); diff --git a/cpp/unittest/server/test_config.cpp b/cpp/unittest/server/test_config.cpp index ef708a8382..cee17542b5 100644 --- a/cpp/unittest/server/test_config.cpp +++ b/cpp/unittest/server/test_config.cpp @@ -25,8 +25,6 @@ namespace { -namespace ms = milvus; - static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf"; @@ -37,25 +35,25 @@ static constexpr uint64_t GB = MB * 1024; } // namespace TEST(ConfigTest, CONFIG_TEST) { - ms::server::ConfigMgr *config_mgr = ms::server::ConfigMgr::GetInstance(); + milvus::server::ConfigMgr *config_mgr = milvus::server::ConfigMgr::GetInstance(); - ms::ErrorCode err = config_mgr->LoadConfigFile(""); - ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR); + milvus::ErrorCode err = config_mgr->LoadConfigFile(""); + ASSERT_EQ(err, milvus::SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(LOG_FILE_PATH); - ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR); + ASSERT_EQ(err, milvus::SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_EQ(err, ms::SERVER_SUCCESS); + ASSERT_EQ(err, milvus::SERVER_SUCCESS); config_mgr->Print(); - ms::server::ConfigNode &root_config = config_mgr->GetRootNode(); - ms::server::ConfigNode &server_config = root_config.GetChild("server_config"); - ms::server::ConfigNode &db_config = root_config.GetChild("db_config"); - ms::server::ConfigNode &metric_config = root_config.GetChild("metric_config"); - ms::server::ConfigNode &cache_config = root_config.GetChild("cache_config"); - ms::server::ConfigNode invalid_config = root_config.GetChild("invalid_config"); + milvus::server::ConfigNode &root_config = config_mgr->GetRootNode(); + milvus::server::ConfigNode &server_config = root_config.GetChild("server_config"); + milvus::server::ConfigNode &db_config = root_config.GetChild("db_config"); + milvus::server::ConfigNode &metric_config = root_config.GetChild("metric_config"); + milvus::server::ConfigNode &cache_config = root_config.GetChild("cache_config"); + milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config"); auto valus = invalid_config.GetSequence("not_exist"); float ff = invalid_config.GetFloatValue("not_exist", 3.0); ASSERT_EQ(ff, 3.0); @@ -69,10 +67,10 @@ TEST(ConfigTest, CONFIG_TEST) { double test = server_config.GetDoubleValue("test"); ASSERT_EQ(test, 2.5); - ms::server::ConfigNode fake; + milvus::server::ConfigNode fake; server_config.AddChild("fake", fake); fake = server_config.GetChild("fake"); - ms::server::ConfigNodeArr arr; + milvus::server::ConfigNodeArr arr; server_config.GetChildren(arr); ASSERT_EQ(arr.size(), 1UL); @@ -89,7 +87,7 @@ TEST(ConfigTest, CONFIG_TEST) { auto seq = server_config.GetSequence("seq"); ASSERT_EQ(seq.size(), 2UL); - ms::server::ConfigNode combine; + milvus::server::ConfigNode combine; combine.Combine(server_config); combine.PrintAll(); @@ -102,8 +100,8 @@ TEST(ConfigTest, CONFIG_TEST) { } TEST(ConfigTest, SERVER_CONFIG_TEST) { - ms::server::Config &config = ms::server::Config::GetInstance(); - ms::Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + milvus::server::Config &config = milvus::server::Config::GetInstance(); + milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH); ASSERT_TRUE(s.ok()); s = config.ValidateConfig(); diff --git a/cpp/unittest/server/test_rpc.cpp b/cpp/unittest/server/test_rpc.cpp index b847ec3116..7d3e0b5511 100644 --- a/cpp/unittest/server/test_rpc.cpp +++ b/cpp/unittest/server/test_rpc.cpp @@ -36,8 +36,6 @@ namespace { -namespace ms = milvus; - static const char *TABLE_NAME = "test_grpc"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t INDEX_FILE_SIZE = 1024; @@ -49,29 +47,29 @@ class RpcHandlerTest : public testing::Test { protected: void SetUp() override { - auto res_mgr = ms::scheduler::ResMgrInst::GetInstance(); + auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = ms::scheduler::Connection("IO", 500.0); - auto PCIE = ms::scheduler::Connection("IO", 11000.0); + auto default_conn = milvus::scheduler::Connection("IO", 500.0); + auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - ms::scheduler::SchedInst::GetInstance()->Start(); - ms::scheduler::JobMgrInst::GetInstance()->Start(); + milvus::scheduler::SchedInst::GetInstance()->Start(); + milvus::scheduler::JobMgrInst::GetInstance()->Start(); - ms::engine::DBOptions opt; + milvus::engine::DBOptions opt; - ms::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); - ms::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); - ms::server::Config::GetInstance().SetDBConfigSecondaryPath(""); - ms::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); - ms::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); - ms::server::Config::GetInstance().SetCacheConfigCacheInsertData(""); - ms::server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); + milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); + milvus::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); + milvus::server::Config::GetInstance().SetDBConfigSecondaryPath(""); + milvus::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); + milvus::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); + milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData(""); + milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); @@ -81,11 +79,11 @@ class RpcHandlerTest : public testing::Test { // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - ms::server::Config::GetInstance().SetResourceConfigMode("single"); - ms::server::DBWrapper::GetInstance().StartService(); + milvus::server::Config::GetInstance().SetResourceConfigMode("single"); + milvus::server::DBWrapper::GetInstance().StartService(); //initialize handler, create table - handler = std::make_shared(); + handler = std::make_shared(); ::grpc::ServerContext context; ::milvus::grpc::TableSchema request; ::milvus::grpc::Status status; @@ -98,15 +96,15 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { - ms::server::DBWrapper::GetInstance().StopService(); - ms::scheduler::JobMgrInst::GetInstance()->Stop(); - ms::scheduler::ResMgrInst::GetInstance()->Stop(); - ms::scheduler::SchedInst::GetInstance()->Stop(); + milvus::server::DBWrapper::GetInstance().StopService(); + milvus::scheduler::JobMgrInst::GetInstance()->Stop(); + milvus::scheduler::ResMgrInst::GetInstance()->Stop(); + milvus::scheduler::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } protected: - std::shared_ptr handler; + std::shared_ptr handler; }; void @@ -425,16 +423,16 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) { ////////////////////////////////////////////////////////////////////// namespace { -class DummyTask : public ms::server::grpc::GrpcBaseTask { +class DummyTask : public milvus::server::grpc::GrpcBaseTask { public: - ms::Status + milvus::Status OnExecute() override { - return ms::Status::OK(); + return milvus::Status::OK(); } - static ms::server::grpc::BaseTaskPtr + static milvus::server::grpc::BaseTaskPtr Create(std::string &dummy) { - return std::shared_ptr(new DummyTask(dummy)); + return std::shared_ptr(new DummyTask(dummy)); } public: @@ -459,16 +457,16 @@ TEST_F(RpcSchedulerTest, BASE_TASK_TEST) { auto status = task_ptr->Execute(); ASSERT_TRUE(status.ok()); - ms::server::grpc::GrpcRequestScheduler::GetInstance().Start(); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - ms::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + milvus::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); - ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); task_ptr = nullptr; - ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - ms::server::grpc::GrpcRequestScheduler::GetInstance().Stop(); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 0ee214a7e7..7ae1b31711 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -33,12 +33,10 @@ namespace { -namespace ms = milvus; - -static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf"; +static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf"; void -CopyStatus(ms::Status &st1, ms::Status &st2) { +CopyStatus(milvus::Status& st1, milvus::Status& st2) { st1 = st2; } @@ -46,61 +44,61 @@ CopyStatus(ms::Status &st1, ms::Status &st2) { TEST(UtilTest, EXCEPTION_TEST) { std::string err_msg = "failed"; - ms::server::ServerException ex(ms::SERVER_UNEXPECTED_ERROR, err_msg); - ASSERT_EQ(ex.error_code(), ms::SERVER_UNEXPECTED_ERROR); + milvus::server::ServerException ex(milvus::SERVER_UNEXPECTED_ERROR, err_msg); + ASSERT_EQ(ex.error_code(), milvus::SERVER_UNEXPECTED_ERROR); std::string msg = ex.what(); ASSERT_EQ(msg, err_msg); } TEST(UtilTest, SIGNAL_TEST) { - ms::server::SignalUtil::PrintStacktrace(); + milvus::server::SignalUtil::PrintStacktrace(); } TEST(UtilTest, COMMON_TEST) { uint64_t total_mem = 0, free_mem = 0; - ms::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); + milvus::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); uint32_t thread_cnt = 0; - ms::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); + milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); std::string path1 = "/tmp/milvus_test/"; std::string path2 = path1 + "common_test_12345/"; std::string path3 = path2 + "abcdef"; - ms::Status status = ms::server::CommonUtil::CreateDirectory(path3); + milvus::Status status = milvus::server::CommonUtil::CreateDirectory(path3); ASSERT_TRUE(status.ok()); //test again - status = ms::server::CommonUtil::CreateDirectory(path3); + status = milvus::server::CommonUtil::CreateDirectory(path3); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(ms::server::CommonUtil::IsDirectoryExist(path3)); + ASSERT_TRUE(milvus::server::CommonUtil::IsDirectoryExist(path3)); - status = ms::server::CommonUtil::DeleteDirectory(path1); + status = milvus::server::CommonUtil::DeleteDirectory(path1); ASSERT_TRUE(status.ok()); //test again - status = ms::server::CommonUtil::DeleteDirectory(path1); + status = milvus::server::CommonUtil::DeleteDirectory(path1); ASSERT_TRUE(status.ok()); - ASSERT_FALSE(ms::server::CommonUtil::IsDirectoryExist(path1)); - ASSERT_FALSE(ms::server::CommonUtil::IsFileExist(path1)); + ASSERT_FALSE(milvus::server::CommonUtil::IsDirectoryExist(path1)); + ASSERT_FALSE(milvus::server::CommonUtil::IsFileExist(path1)); - std::string exe_path = ms::server::CommonUtil::GetExePath(); + std::string exe_path = milvus::server::CommonUtil::GetExePath(); ASSERT_FALSE(exe_path.empty()); time_t tt; time(&tt); tm time_struct; memset(&time_struct, 0, sizeof(tm)); - ms::server::CommonUtil::ConvertTime(tt, time_struct); + milvus::server::CommonUtil::ConvertTime(tt, time_struct); ASSERT_GT(time_struct.tm_year, 0); ASSERT_GT(time_struct.tm_mon, 0); ASSERT_GT(time_struct.tm_mday, 0); - ms::server::CommonUtil::ConvertTime(time_struct, tt); + milvus::server::CommonUtil::ConvertTime(time_struct, tt); ASSERT_GT(tt, 0); - bool res = ms::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); + bool res = milvus::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); ASSERT_EQ(time_struct.tm_year, 119); ASSERT_EQ(time_struct.tm_mon, 2); ASSERT_EQ(time_struct.tm_mday, 23); @@ -110,38 +108,38 @@ TEST(UtilTest, COMMON_TEST) { TEST(UtilTest, STRINGFUNCTIONS_TEST) { std::string str = " test zilliz"; - ms::server::StringHelpFunctions::TrimStringBlank(str); + milvus::server::StringHelpFunctions::TrimStringBlank(str); ASSERT_EQ(str, "test zilliz"); str = "\"test zilliz\""; - ms::server::StringHelpFunctions::TrimStringQuote(str, "\""); + milvus::server::StringHelpFunctions::TrimStringQuote(str, "\""); ASSERT_EQ(str, "test zilliz"); str = "a,b,c"; std::vector result; - auto status = ms::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); + auto status = milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); + status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result); + status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); str = "55,\"aa,gg,yy\",b"; result.clear(); - status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); + status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); } TEST(UtilTest, BLOCKINGQUEUE_TEST) { - ms::server::BlockingQueue bq; + milvus::server::BlockingQueue bq; static const size_t count = 10; bq.SetCapacity(count); @@ -170,13 +168,13 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { - auto status = ms::server::InitLog(LOG_FILE_PATH); + auto status = milvus::server::InitLog(LOG_FILE_PATH); ASSERT_TRUE(status.ok()); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); - std::string fname = ms::server::CommonUtil::GetFileName(LOG_FILE_PATH); + std::string fname = milvus::server::CommonUtil::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); } @@ -185,39 +183,39 @@ TEST(UtilTest, TIMERECORDER_TEST) { if (log_level == 5) { continue; //skip fatal } - ms::TimeRecorder rc("time", log_level); + milvus::TimeRecorder rc("time", log_level); rc.RecordSection("end"); } } TEST(UtilTest, STATUS_TEST) { - auto status = ms::Status::OK(); + auto status = milvus::Status::OK(); std::string str = status.ToString(); ASSERT_FALSE(str.empty()); - status = ms::Status(ms::DB_ERROR, "mistake"); - ASSERT_EQ(status.code(), ms::DB_ERROR); + status = milvus::Status(milvus::DB_ERROR, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_ERROR); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = ms::Status(ms::DB_NOT_FOUND, "mistake"); - ASSERT_EQ(status.code(), ms::DB_NOT_FOUND); + status = milvus::Status(milvus::DB_NOT_FOUND, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = ms::Status(ms::DB_ALREADY_EXIST, "mistake"); - ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); + status = milvus::Status(milvus::DB_ALREADY_EXIST, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = ms::Status(ms::DB_META_TRANSACTION_FAILED, "mistake"); - ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED); + status = milvus::Status(milvus::DB_META_TRANSACTION_FAILED, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED); str = status.ToString(); ASSERT_FALSE(str.empty()); - auto status_copy = ms::Status::OK(); + auto status_copy = milvus::Status::OK(); CopyStatus(status_copy, status); - ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED); + ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED); auto status_ref(status); ASSERT_EQ(status_ref.code(), status.code()); @@ -230,120 +228,126 @@ TEST(UtilTest, STATUS_TEST) { TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; - auto status = ms::server::ValidationUtil::ValidateTableName(table_name); + auto status = milvus::server::ValidationUtil::ValidateTableName(table_name); ASSERT_TRUE(status.ok()); table_name = "12sds"; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); table_name = ""; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_SUCCESS); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_SUCCESS); table_name = "!@#!@"; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); table_name = "_!@#!@"; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); table_name = "中文"; - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); table_name = std::string(10000, 'a'); - status = ms::server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); + status = milvus::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(-1).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(0).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16385).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16384).code(), ms::SERVER_SUCCESS); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(1).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(-1).code(), + milvus::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(0).code(), + milvus::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(16385).code(), + milvus::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(16384).code(), milvus::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(1).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::INVALID).code(), - ms::SERVER_INVALID_INDEX_TYPE); - for (int i = 1; i <= (int) ms::engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType(i).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(), + milvus::SERVER_INVALID_INDEX_TYPE); + for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) { + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS); } - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::MAX_VALUE + 1).code(), - ms::SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType( + (int)milvus::engine::EngineType::MAX_VALUE + 1).code(), + milvus::SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(0).code(), ms::SERVER_INVALID_INDEX_NLIST); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(100).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(0).code(), milvus::SERVER_INVALID_INDEX_NLIST); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(100).code(), milvus::SERVER_SUCCESS); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), ms::SERVER_INVALID_INDEX_FILE_SIZE); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), + milvus::SERVER_INVALID_INDEX_FILE_SIZE); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), milvus::SERVER_SUCCESS); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(0).code(), ms::SERVER_INVALID_INDEX_METRIC_TYPE); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), ms::SERVER_SUCCESS); - ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(0).code(), + milvus::SERVER_INVALID_INDEX_METRIC_TYPE); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), milvus::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { - ms::engine::meta::TableSchema schema; - ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), ms::SERVER_SUCCESS); + milvus::engine::meta::TableSchema schema; + ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) { - ms::engine::meta::TableSchema schema; + milvus::engine::meta::TableSchema schema; schema.nlist_ = 100; - ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_GPU_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateGpuIndex(0).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateGpuIndex(100).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(100).code(), milvus::SERVER_SUCCESS); size_t memory = 0; - ASSERT_EQ(ms::server::ValidationUtil::GetGpuMemory(0, memory).code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::GetGpuMemory(100, memory).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::GetGpuMemory(0, memory).code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::GetGpuMemory(100, memory).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateIpAddress("not ip").code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsNumber("1234").code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsNumber("not number").code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsNumber("1234").code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("not number").code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) { std::string str = "true"; - ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS); str = "not bool"; - ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsFloat("not double").code(), ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsFloat("not double").code(), milvus::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) { - ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("not uri").code(), ms::SERVER_SUCCESS); - ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), - ms::SERVER_SUCCESS); - ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), - ms::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("not uri").code(), milvus::SERVER_SUCCESS); + ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), + milvus::SERVER_SUCCESS); + ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), + milvus::SERVER_SUCCESS); } TEST(UtilTest, ROLLOUTHANDLER_TEST) { @@ -374,7 +378,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) { file.open(tmp.c_str()); file << "zilliz" << std::endl; - ms::server::RolloutHandler(tmp.c_str(), 0, list[i]); + milvus::server::RolloutHandler(tmp.c_str(), 0, list[i]); tmp.append(".1"); std::ifstream file2; diff --git a/cpp/unittest/wrapper/test_wrapper.cpp b/cpp/unittest/wrapper/test_wrapper.cpp index 4dd2e33bd5..fe8cc3d914 100644 --- a/cpp/unittest/wrapper/test_wrapper.cpp +++ b/cpp/unittest/wrapper/test_wrapper.cpp @@ -42,30 +42,30 @@ constexpr int64_t DEVICE_ID = 0; class ParamGenerator { public: - static ParamGenerator &GetInstance() { + static ParamGenerator& GetInstance() { static ParamGenerator instance; return instance; } - kw::Config Gen(const ms::IndexType &type) { + knowhere::Config Gen(const milvus::engine::IndexType& type) { switch (type) { - case ms::IndexType::FAISS_IDMAP: { + case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared(); tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case ms::IndexType::FAISS_IVFFLAT_CPU: - case ms::IndexType::FAISS_IVFFLAT_GPU: - case ms::IndexType::FAISS_IVFFLAT_MIX: { + case milvus::engine::IndexType::FAISS_IVFFLAT_CPU: + case milvus::engine::IndexType::FAISS_IVFFLAT_GPU: + case milvus::engine::IndexType::FAISS_IVFFLAT_MIX: { auto tempconf = std::make_shared(); tempconf->nlist = 100; tempconf->nprobe = 16; tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case ms::IndexType::FAISS_IVFSQ8_CPU: - case ms::IndexType::FAISS_IVFSQ8_GPU: - case ms::IndexType::FAISS_IVFSQ8_MIX: { + case milvus::engine::IndexType::FAISS_IVFSQ8_CPU: + case milvus::engine::IndexType::FAISS_IVFSQ8_GPU: + case milvus::engine::IndexType::FAISS_IVFSQ8_MIX: { auto tempconf = std::make_shared(); tempconf->nlist = 100; tempconf->nprobe = 16; @@ -73,8 +73,8 @@ class ParamGenerator { tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case ms::IndexType::FAISS_IVFPQ_CPU: - case ms::IndexType::FAISS_IVFPQ_GPU: { + case milvus::engine::IndexType::FAISS_IVFPQ_CPU: + case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { auto tempconf = std::make_shared(); tempconf->nlist = 100; tempconf->nprobe = 16; @@ -83,7 +83,7 @@ class ParamGenerator { tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case ms::IndexType::NSG_MIX: { + case milvus::engine::IndexType::NSG_MIX: { auto tempconf = std::make_shared(); tempconf->nlist = 100; tempconf->nprobe = 16; @@ -100,13 +100,13 @@ class ParamGenerator { }; class KnowhereWrapperTest - : public TestWithParam<::std::tuple> { + : public TestWithParam<::std::tuple> { protected: void SetUp() override { knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, - 1024 * 1024 * 200, - 1024 * 1024 * 300, - 2); + 1024 * 1024 * 200, + 1024 * 1024 * 300, + 2); std::string generator_type; std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam(); @@ -126,7 +126,7 @@ class KnowhereWrapperTest knowhere::FaissGpuResourceMgr::GetInstance().Free(); } - void AssertResult(const std::vector &ids, const std::vector &dis) { + void AssertResult(const std::vector& ids, const std::vector& dis) { EXPECT_EQ(ids.size(), nq * k); EXPECT_EQ(dis.size(), nq * k); @@ -153,8 +153,8 @@ class KnowhereWrapperTest } protected: - ms::IndexType index_type; - kw::Config conf; + milvus::engine::IndexType index_type; + knowhere::Config conf; int dim = DIM; int nb = NB; @@ -164,7 +164,7 @@ class KnowhereWrapperTest std::vector xq; std::vector ids; - ms::VecIndexPtr index_ = nullptr; + milvus::engine::VecIndexPtr index_ = nullptr; // Ground Truth std::vector gt_ids; @@ -174,15 +174,25 @@ class KnowhereWrapperTest INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, Values( //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] - std::make_tuple(ms::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, + "Default", + 64, + 100000, + 10, + 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, + "Default", + 64, + 100000, + 10, + 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), // std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), - std::make_tuple(ms::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) + std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) ) ); @@ -220,7 +230,7 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) { { std::string file_location = "/tmp/knowhere_gpu_file"; write_index(index_, file_location); - auto new_index = ms::read_index(file_location); + auto new_index = milvus::engine::read_index(file_location); auto dev_idx = new_index->CopyToGpu(DEVICE_ID); for (int i = 0; i < 10; ++i) { @@ -261,7 +271,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { { std::string file_location = "/tmp/knowhere"; write_index(index_, file_location); - auto new_index = ms::read_index(file_location); + auto new_index = milvus::engine::read_index(file_location); EXPECT_EQ(new_index->GetType(), ConvertToCpuIndexType(index_type)); EXPECT_EQ(new_index->Dimension(), index_->Dimension()); EXPECT_EQ(new_index->Count(), index_->Count()); From 8a9e7606394dbbb58cd6586e44f5208ccb8c25ef Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 11 Oct 2019 16:18:19 +0800 Subject: [PATCH 2/6] refine code Former-commit-id: 7935a4ed21d423e5fc72adc8ab68b7f69dae554e --- .../knowhere/index/vector_index/nsg/NSG.cpp | 4 +- cpp/src/core/thirdparty/SPTAG/build.sh | 6 - cpp/src/core/unittest/test_json.cpp | 28 ----- cpp/src/core/unittest/test_nsg.cpp | 107 ------------------ cpp/unittest/server/util_test.cpp | 12 +- 5 files changed, 8 insertions(+), 149 deletions(-) delete mode 100644 cpp/src/core/unittest/test_json.cpp delete mode 100644 cpp/src/core/unittest/test_nsg.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 375664fbf8..bdf538c204 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -765,8 +765,8 @@ NsgIndex::SetKnnGraph(Graph& g) { } // void NsgIndex::GetKnnGraphFromFile() { -// //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph"; -// std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph"; +// //std::string filename = "sift.1M.50NN.graph"; +// std::string filename = "sift.50NN.graph"; // // std::ifstream in(filename, std::ios::binary); // unsigned k; diff --git a/cpp/src/core/thirdparty/SPTAG/build.sh b/cpp/src/core/thirdparty/SPTAG/build.sh index 9b83643df2..bd9833d605 100755 --- a/cpp/src/core/thirdparty/SPTAG/build.sh +++ b/cpp/src/core/thirdparty/SPTAG/build.sh @@ -9,12 +9,6 @@ cd cmake_build INSTALL_PREFIX=$(pwd)/../../build -BOOST_PATH="/home/zilliz/opt/app/boost" -TBB_PATH="/home/zilliz/opt/app/tbb/tbb" -OPTION="-DBOOST_ROOT=$BOOST_PATH -DTBB_DIR=${TBB_PATH}" - -# CMAKE_CMD="cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} ${OPTION} ../" - CMAKE_CMD="cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} ../" ${CMAKE_CMD} diff --git a/cpp/src/core/unittest/test_json.cpp b/cpp/src/core/unittest/test_json.cpp deleted file mode 100644 index a3babbc1b7..0000000000 --- a/cpp/src/core/unittest/test_json.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "knowhere/common/config.h" - -int -main() { - knowhere::Config cfg; - - cfg["size"] = size_t(199); - auto size = cfg.get_with_default("size", 123); - auto size_2 = cfg["size"].as(); - printf("%d", size_2); -} diff --git a/cpp/src/core/unittest/test_nsg.cpp b/cpp/src/core/unittest/test_nsg.cpp deleted file mode 100644 index a4b6006e71..0000000000 --- a/cpp/src/core/unittest/test_nsg.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include "knowhere/index/index.h" -#include "test/utils.h" - -//#include - -void -load_data(std::string& filename, float*& data, unsigned& num, - unsigned& dim) { // load data with sift10K pattern - std::ifstream in(filename, std::ios::binary); - if (!in.is_open()) { - std::cout << "open file error" << std::endl; - exit(-1); - } - in.read((char*)&dim, 4); - in.seekg(0, std::ios::end); - std::ios::pos_type ss = in.tellg(); - size_t fsize = (size_t)ss; - num = (unsigned)(fsize / (dim + 1) / 4); - data = new float[(size_t)num * (size_t)dim]; - - in.seekg(0, std::ios::beg); - for (size_t i = 0; i < num; i++) { - in.seekg(4, std::ios::cur); - in.read((char*)(data + i * dim), dim * 4); - } - in.close(); -} - -void -test_distance() { - std::vector xb{1, 2, 3, 4}; - std::vector xq{2, 2, 3, 4}; - float r = calculate(xb.data(), xq.data(), 4); - std::cout << r << std::endl; -} - -int -main() { - test_distance(); - - BuildParams params; - params.search_length = 100; - params.candidate_pool_size = 100; - params.out_degree = 50; - - float* data = nullptr; - int64_t* ids = nullptr; - unsigned ntotal, dim; - std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/siftsmall/siftsmall_base.fvecs"; - // std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift/sift_base.fvecs"; - - load_data(filename, data, ntotal, dim); - assert(data); - // float x = calculate(data + dim * 0, data + dim * 62, dim); - // std::cout << x << std::endl; - - NsgIndex index(dim, ntotal); - - auto s = std::chrono::high_resolution_clock::now(); - index.Build_with_ids(ntotal, data, ids, params); - auto e = std::chrono::high_resolution_clock::now(); - std::chrono::duration diff = e - s; - std::cout << "indexing time: " << diff.count() << "\n"; - - int k = 10; - int nq = 1000; - SearchParams s_params; - s_params.search_length = 50; - auto dist = new float[nq * k]; - auto ids_b = new int64_t[nq * k]; - s = std::chrono::high_resolution_clock::now(); - // ProfilerStart("xx.prof"); - index.Search(data, nq, dim, k, dist, ids_b, s_params); - // ProfilerStop(); - e = std::chrono::high_resolution_clock::now(); - diff = e - s; - std::cout << "search time: " << diff.count() << "\n"; - - for (int i = 0; i < k; ++i) { - std::cout << "id " << ids_b[i] << std::endl; - // std::cout << "dist " << dist[i] << std::endl; - } - - delete[] dist; - delete[] ids_b; - - return 0; -} diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 7ae1b31711..395839a8c0 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -107,13 +107,13 @@ TEST(UtilTest, COMMON_TEST) { } TEST(UtilTest, STRINGFUNCTIONS_TEST) { - std::string str = " test zilliz"; + std::string str = " test str"; milvus::server::StringHelpFunctions::TrimStringBlank(str); - ASSERT_EQ(str, "test zilliz"); + ASSERT_EQ(str, "test str"); - str = "\"test zilliz\""; + str = "\"test str\""; milvus::server::StringHelpFunctions::TrimStringQuote(str, "\""); - ASSERT_EQ(str, "test zilliz"); + ASSERT_EQ(str, "test str"); str = "a,b,c"; std::vector result; @@ -376,7 +376,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) { std::ofstream file; file.open(tmp.c_str()); - file << "zilliz" << std::endl; + file << "test" << std::endl; milvus::server::RolloutHandler(tmp.c_str(), 0, list[i]); @@ -386,7 +386,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) { std::string tmp2; file2 >> tmp2; - ASSERT_EQ(tmp2, "zilliz"); + ASSERT_EQ(tmp2, "test"); } boost::filesystem::remove_all(dir2); } From 59f45763d33bae07cccc9ac946caac9dfa647cb6 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 11 Oct 2019 17:38:48 +0800 Subject: [PATCH 3/6] MS-636 Add optimizer in scheduler for FAISS_IVFSQ8H Former-commit-id: 4dff6236c98a86965e686b4a94e5492e0c1d5722 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 25 +++++------ cpp/src/scheduler/Algorithm.cpp | 2 - cpp/src/scheduler/JobMgr.cpp | 7 ++- cpp/src/scheduler/SchedInst.cpp | 3 ++ cpp/src/scheduler/SchedInst.h | 24 +++++++++++ cpp/src/scheduler/TaskCreator.cpp | 4 +- .../scheduler/action/PushTaskToNeighbour.cpp | 43 ++++++++++++------- cpp/src/scheduler/optimizer/HybridPass.cpp | 7 ++- cpp/src/scheduler/optimizer/Optimizer.h | 6 ++- cpp/src/scheduler/task/BuildIndexTask.cpp | 5 +++ cpp/unittest/CMakeLists.txt | 2 + cpp/unittest/db/CMakeLists.txt | 9 ++++ cpp/unittest/db/appendix/log_config.conf | 27 ++++++++++++ cpp/unittest/db/appendix/server_config.yaml | 37 ++++++++++++++++ cpp/unittest/db/test_db.cpp | 26 +++++++++++ cpp/unittest/db/utils.cpp | 2 +- 17 files changed, 193 insertions(+), 37 deletions(-) create mode 100644 cpp/unittest/db/appendix/log_config.conf create mode 100644 cpp/unittest/db/appendix/server_config.yaml diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1a62a791b0..d0763d5c06 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA. ## New Feature - MS-627 - Integrate new index: IVFSQHybrid - MS-631 - IVFSQ8H Index support +- MS-636 - Add optimizer in scheduler for FAISS_IVFSQ8H ## Task - MS-554 - Change license to Apache 2.0 diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f81fb32e7f..cf8f1824d2 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -900,20 +900,21 @@ DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; - scheduler::BuildIndexJobPtr job = std::make_shared(0, meta_ptr_, options_); + if (!to_index_files.empty()) { + scheduler::BuildIndexJobPtr job = std::make_shared(0, meta_ptr_, options_); - // step 2: put build index task to scheduler - for (auto& file : to_index_files) { - scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); - job->AddToIndexFiles(file_ptr); + // step 2: put build index task to scheduler + for (auto& file : to_index_files) { + scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); + job->AddToIndexFiles(file_ptr); + } + scheduler::JobMgrInst::GetInstance()->Put(job); + job->WaitBuildIndexFinish(); + if (!job->GetStatus().ok()) { + Status status = job->GetStatus(); + ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); + } } - scheduler::JobMgrInst::GetInstance()->Put(job); - job->WaitBuildIndexFinish(); - if (!job->GetStatus().ok()) { - Status status = job->GetStatus(); - ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); - } - // for (auto &file : to_index_files) { // status = BuildIndex(file); // if (!status.ok()) { diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index 44f83742c2..b2156b3f97 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -29,8 +29,6 @@ constexpr uint64_t MAXINT = std::numeric_limits::max(); uint64_t ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr, std::vector& path) { - std::vector> paths; - uint64_t num_of_resources = res_mgr->GetAllResources().size(); std::unordered_map id_name_map; std::unordered_map name_id_map; diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index a4ef83ad75..170dee4b80 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -16,7 +16,9 @@ // under the License. #include "scheduler/JobMgr.h" +#include "SchedInst.h" #include "TaskCreator.h" +#include "optimizer/Optimizer.h" #include "task/Task.h" #include @@ -67,8 +69,9 @@ JobMgr::worker_function() { } auto tasks = build_task(job); - - // TODO: optimizer all task + for (auto& task : tasks) { + OptimizerInst::GetInstance()->Run(task); + } // disk resources NEVER be empty. if (auto disk = res_mgr_->GetDiskResources()[0].lock()) { diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index b9edbca001..cc2b4e280a 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -38,6 +38,9 @@ std::mutex SchedInst::mutex_; scheduler::JobMgrPtr JobMgrInst::instance = nullptr; std::mutex JobMgrInst::mutex_; +OptimizerPtr OptimizerInst::instance = nullptr; +std::mutex OptimizerInst::mutex_; + void load_simple_config() { server::Config& config = server::Config::GetInstance(); diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index dc8e4ed478..0d2a04b02c 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -20,9 +20,12 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" +#include "optimizer/HybridPass.h" +#include "optimizer/Optimizer.h" #include #include +#include namespace milvus { namespace scheduler { @@ -81,6 +84,27 @@ class JobMgrInst { static std::mutex mutex_; }; +class OptimizerInst { + public: + static OptimizerPtr + GetInstance() { + if (instance == nullptr) { + std::lock_guard lock(mutex_); + if (instance == nullptr) { + HybridPassPtr pass_ptr = std::make_shared(); + std::vector pass_list; + pass_list.push_back(pass_ptr); + instance = std::make_shared(pass_list); + } + } + return instance; + } + + private: + static scheduler::OptimizerPtr instance; + static std::mutex mutex_; +}; + void StartSchedulerService(); diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index ee63c2c6b7..40cfa9aac6 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -16,10 +16,10 @@ // under the License. #include "scheduler/TaskCreator.h" -#include #include "SchedInst.h" -#include "scheduler/tasklabel/BroadcastLabel.h" +#include "tasklabel/BroadcastLabel.h" #include "tasklabel/DefaultLabel.h" +#include "tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 53dd45faca..95f8212297 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -19,6 +19,7 @@ #include #include "../Algorithm.h" #include "Action.h" +#include "scheduler/tasklabel/SpecResLabel.h" #include "src/cache/GpuCacheMgr.h" #include "src/server/Config.h" @@ -145,25 +146,35 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr paths.emplace_back(path); } if (task->job_.lock()->type() == JobType::SEARCH) { - // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost - uint64_t min_cost = std::numeric_limits::max(); - uint64_t min_cost_idx = 0; - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (compute_resources[i]->TotalTasks() == 0) { - min_cost_idx = i; - break; - } - uint64_t cost = - compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + transport_costs[i]; - if (min_cost > cost) { - min_cost = cost; - min_cost_idx = i; + auto label = task->label(); + auto spec_label = std::static_pointer_cast(label); + if (spec_label->resource().lock()->type() == ResourceType::CPU) { + std::vector spec_path; + spec_path.push_back(spec_label->resource().lock()->name()); + spec_path.push_back(resource->name()); + task->path() = Path(spec_path, spec_path.size() - 1); + } else { + // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost + uint64_t min_cost = std::numeric_limits::max(); + uint64_t min_cost_idx = 0; + for (uint64_t i = 0; i < compute_resources.size(); ++i) { + if (compute_resources[i]->TotalTasks() == 0) { + min_cost_idx = i; + break; + } + uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + + transport_costs[i]; + if (min_cost > cost) { + min_cost = cost; + min_cost_idx = i; + } } + + // step 3: set path in task + Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); + task->path() = task_path; } - // step 3: set path in task - Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); - task->path() = task_path; } else if (task->job_.lock()->type() == JobType::BUILD) { // step2: Read device id in config // get build index gpu resource diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp index 343d6ec81e..d63fc2e819 100644 --- a/cpp/src/scheduler/optimizer/HybridPass.cpp +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -16,7 +16,9 @@ // under the License. #include "scheduler/optimizer/HybridPass.h" +#include "scheduler/SchedInst.h" #include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { @@ -28,7 +30,10 @@ HybridPass::Run(const TaskPtr& task) { return false; auto search_task = std::static_pointer_cast(task); if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: make specified label + // TODO: remove "cpu" hardcode + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; return true; } return false; diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h index 99282e66a6..68b519e115 100644 --- a/cpp/src/scheduler/optimizer/Optimizer.h +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "Pass.h" @@ -34,7 +35,8 @@ namespace scheduler { class Optimizer { public: - Optimizer() = default; + explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { + } void Init(); @@ -46,5 +48,7 @@ class Optimizer { std::vector pass_list_; }; +using OptimizerPtr = std::shared_ptr; + } // namespace scheduler } // namespace milvus diff --git a/cpp/src/scheduler/task/BuildIndexTask.cpp b/cpp/src/scheduler/task/BuildIndexTask.cpp index f2cebcac9e..25d3d73a7b 100644 --- a/cpp/src/scheduler/task/BuildIndexTask.cpp +++ b/cpp/src/scheduler/task/BuildIndexTask.cpp @@ -124,6 +124,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = status; + to_index_engine_ = nullptr; return; } @@ -136,6 +137,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + to_index_engine_ = nullptr; return; } } catch (std::exception& ex) { @@ -150,6 +152,7 @@ XBuildIndexTask::Execute() { << std::endl; build_index_job->GetStatus() = Status(DB_ERROR, msg); + to_index_engine_ = nullptr; return; } @@ -158,6 +161,7 @@ XBuildIndexTask::Execute() { meta_ptr->HasTable(file_->table_id_, has_table); if (!has_table) { meta_ptr->DeleteTableFiles(file_->table_id_); + to_index_engine_ = nullptr; return; } @@ -177,6 +181,7 @@ XBuildIndexTask::Execute() { << ", possible out of disk space" << std::endl; build_index_job->GetStatus() = Status(DB_ERROR, msg); + to_index_engine_ = nullptr; return; } diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 6c9aeadcd1..ac4fae85bb 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -50,6 +50,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer scheduler_optimizer_files) set(scheduler_files ${scheduler_main_files} ${scheduler_action_files} @@ -57,6 +58,7 @@ set(scheduler_files ${scheduler_job_files} ${scheduler_resource_files} ${scheduler_task_files} + ${scheduler_optimizer_files} ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 2cbf55a208..4bce9f35b3 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -31,3 +31,12 @@ target_link_libraries(test_db install(TARGETS test_db DESTINATION unittest) +configure_file(appendix/server_config.yaml + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml" + COPYONLY) + +configure_file(appendix/log_config.conf + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf" + COPYONLY) + + diff --git a/cpp/unittest/db/appendix/log_config.conf b/cpp/unittest/db/appendix/log_config.conf new file mode 100644 index 0000000000..0a3e0d21af --- /dev/null +++ b/cpp/unittest/db/appendix/log_config.conf @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB +* DEBUG: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" +* TRACE: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/cpp/unittest/db/appendix/server_config.yaml b/cpp/unittest/db/appendix/server_config.yaml new file mode 100644 index 0000000000..f92b2f1a18 --- /dev/null +++ b/cpp/unittest/db/appendix/server_config.yaml @@ -0,0 +1,37 @@ +# All the following configurations are default values. + +server_config: + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 + +db_config: + primary_path: /tmp/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon + + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' + + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index + +metric_config: + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus used to fetch metrics + +cache_config: + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache + +engine_config: + blas_threshold: 20 + +resource_config: + resource_pool: + - cpu + - gpu0 diff --git a/cpp/unittest/db/test_db.cpp b/cpp/unittest/db/test_db.cpp index e502d0f742..9e80afbc09 100644 --- a/cpp/unittest/db/test_db.cpp +++ b/cpp/unittest/db/test_db.cpp @@ -23,14 +23,18 @@ #include "db/DBFactory.h" #include "cache/CpuCacheMgr.h" #include "utils/CommonUtil.h" +#include "server/Config.h" #include #include #include #include + namespace { +static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; + static const char *TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t VECTOR_COUNT = 25000; @@ -228,6 +232,9 @@ TEST_F(DBTest, DB_TEST) { } TEST_F(DBTest, SEARCH_TEST) { + milvus::server::Config &config = milvus::server::Config::GetInstance(); + milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); @@ -290,6 +297,25 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } + //test FAISS_IVFSQ8H optimizer + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + milvus::engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); + ASSERT_TRUE(stat.ok()); + } + + {//search by specify index file + milvus::engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; + milvus::engine::QueryResults results; + stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); + ASSERT_TRUE(stat.ok()); + } + + // TODO(lxj): add groundTruth assert } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index dfbffc66bc..67beeba36f 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -97,7 +97,7 @@ DBTest::SetUp() { auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); From 5bcb660110bf42daf01d3de5f9b58e8653366a17 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 11 Oct 2019 19:41:43 +0800 Subject: [PATCH 4/6] MS-635 add compile option Former-commit-id: 773d61d34bdb131ae3219ddfbe9e4eb656292230 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 4 ++ cpp/build.sh | 9 +++- .../core/cmake/ThirdPartyPackagesCore.cmake | 25 +++++---- .../index/vector_index/FaissBaseIndex.cpp | 8 +-- .../index/vector_index/IndexIVFSQ.cpp | 5 ++ .../index/vector_index/IndexIVFSQHybrid.cpp | 54 +++++++++++++++++++ .../index/vector_index/IndexIVFSQHybrid.h | 2 + cpp/src/core/unittest/test_ivf.cpp | 12 +++-- cpp/src/wrapper/VecImpl.h | 6 +-- cpp/src/wrapper/VecIndex.cpp | 8 ++- cpp/src/wrapper/VecIndex.h | 2 +- 12 files changed, 110 insertions(+), 26 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1a62a791b0..6fdcfe65c2 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -49,6 +49,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-600 - Reconstruct unittest code - MS-602 - Remove zilliz namespace - MS-610 - Change error code base value from hex to decimal +- MS-635 - Add compile option to support customized faiss # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d2092eb018..cde5d3f90e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -125,6 +125,10 @@ set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) +if (CUSTOMIZATION) + add_definitions(-DCUSTOMIZATION) +endif (CUSTOMIZATION) + include(ExternalProject) include(DefineOptions) include(BuildUtils) diff --git a/cpp/build.sh b/cpp/build.sh index 648206f9ae..e15253772f 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -9,9 +9,12 @@ DB_PATH="/opt/milvus" PROFILING="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" +CUSTOMIZATION="ON" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "p:d:t:ulrcgjh" arg +wget -q --method HEAD + +while getopts "p:d:t:ulrcgjhx" arg do case $arg in p) @@ -45,6 +48,9 @@ do j) USE_JFROG_CACHE="ON" ;; + x) + CUSTOMIZATION="OFF" + ;; h) # help echo " @@ -87,6 +93,7 @@ CMAKE_CMD="cmake \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 5691d4f9dc..f2823a78c6 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -228,17 +228,24 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{KNOWHERE_FAISS_URL}) - set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") -else() +if(CUSTOMIZATION) set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") - message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") + # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") + # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 + # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 + # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 + set(FAISS_MD5 "94988b7bdac4eb82a9575c702a3f2df3") # commit-id 1407526b31cad26f98ceca8dddaface8f18c4c19 branch-0.2.1 + + execute_process(COMMAND wget -q --method HEAD ${FAISS_SOURCE_URL} RESULT_VARIABLE return_code) + message(STATUS "Check the remote cache file ${FAISS_SOURCE_URL}. return code = ${return_code}") + if (NOT return_code EQUAL 0) + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") + endif() +else() + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") + set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135") endif() -# set(FAISS_MD5 "a589663865a8558205533c8ac414278c") -# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 -# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 -# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 -set(FAISS_MD5 "5af237d77947ee632f169bcb36feee2b") # commit-id 2c8affd0da60354e4322fa4c0224519e7912b9c4 branch-0.2.1 +message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") if(DEFINED ENV{KNOWHERE_ARROW_URL}) set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}") diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 2612a63630..783487be3a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -64,17 +64,13 @@ FaissBaseIndex::LoadImpl(const BinarySet& index_binary) { void FaissBaseIndex::SealImpl() { - // TODO(linxj): enable - //#ifdef ZILLIZ_FAISS +#ifdef CUSTOMIZATION faiss::Index* index = index_.get(); auto idx = dynamic_cast(index); if (idx != nullptr) { idx->to_readonly(); } - // else { - // KNOHWERE_ERROR_MSG("Seal failed"); - //} - //#endif +#endif } } // namespace knowhere diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 11e508549a..063dc63550 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -55,10 +55,15 @@ VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); + +#ifdef CUSTOMIZATION faiss::gpu::GpuClonerOptions option; option.allInGpu = true; auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); +#else + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); +#endif std::shared_ptr device_index; device_index.reset(gpu_index); diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index c6c64fdcb4..5449aec860 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -25,6 +25,7 @@ namespace knowhere { +#ifdef CUSTOMIZATION IndexModelPtr IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); @@ -221,4 +222,57 @@ FaissIVFQuantizer::~FaissIVFQuantizer() { // else do nothing } +#else + +QuantizerPtr +IVFSQHybrid::LoadQuantizer(const Config& conf) { + return knowhere::QuantizerPtr(); +} + +void +IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { + +} + +void +IVFSQHybrid::UnsetQuantizer() { + +} + +void +IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { + +} + +IndexModelPtr +IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { + return GPUIVFSQ::Train(dataset, config); +} + +VectorIndexPtr +IVFSQHybrid::CopyGpuToCpu(const Config& config) { + return GPUIVFSQ::CopyGpuToCpu(config); +} + +VectorIndexPtr +IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + return IVF::CopyCpuToGpu(device_id, config); +} + +void +IVFSQHybrid::search_impl(int64_t n, + const float* data, + int64_t k, + float* distances, + int64_t* labels, + const Config& cfg) { + GPUIVF::search_impl(n, data, k, distances, labels, cfg); +} + +void +IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { + GPUIVF::LoadImpl(index_binary); +} + +#endif } // namespace knowhere diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index 73c442093b..e2ca208d90 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -25,12 +25,14 @@ namespace knowhere { +#ifdef CUSTOMIZATION struct FaissIVFQuantizer : public Quantizer { faiss::gpu::GpuIndexFlat* quantizer = nullptr; ~FaissIVFQuantizer() override; }; using FaissIVFQuantizerPtr = std::shared_ptr; +#endif class IVFSQHybrid : public GPUIVFSQ { public: diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index 4e309c3867..cc4712c263 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -70,7 +70,6 @@ enum class ParameterType { ivf, ivfpq, ivfsq, - ivfsqhybrid, nsg, }; @@ -104,7 +103,7 @@ class ParamGenerator { tempconf->nbits = 8; tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; - } else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) { + } else if (type == ParameterType::ivfsq) { auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; @@ -158,8 +157,11 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, // std::make_tuple("IVFPQ", ParameterType::ivfpq), // std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), std::make_tuple("IVFSQ", ParameterType::ivfsq), - std::make_tuple("GPUIVFSQ", ParameterType::ivfsq), - std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid))); +#ifdef CUSTOMIZATION + std::make_tuple("IVFSQHybrid", ParameterType::ivfsq), +#endif + std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)) + ); void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { @@ -558,6 +560,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { } } +#ifdef CUSTOMIZATION TEST_F(GPURESTEST, gpuivfsq) { { // knowhere gpu ivfsq @@ -629,6 +632,7 @@ TEST_F(GPURESTEST, gpuivfsq) { delete search_idx; } } +#endif TEST_F(GPURESTEST, copyandsearch) { // search and copy at the same time diff --git a/cpp/src/wrapper/VecImpl.h b/cpp/src/wrapper/VecImpl.h index 7558201e25..fd9bb79c0a 100644 --- a/cpp/src/wrapper/VecImpl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -17,12 +17,12 @@ #pragma once -#include "VecIndex.h" -#include "knowhere/index/vector_index/VectorIndex.h" - #include #include +#include "VecIndex.h" +#include "knowhere/index/vector_index/VectorIndex.h" + namespace milvus { namespace engine { diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index fe75cbb81b..bd0e8724ad 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include "wrapper/VecIndex.h" -#include "VecImpl.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" @@ -28,6 +26,8 @@ #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" +#include "wrapper/VecIndex.h" +#include "VecImpl.h" #include "utils/Log.h" #include @@ -143,10 +143,12 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { index = std::make_shared(gpu_device); break; } +#ifdef CUSTOMIZATION case IndexType::FAISS_IVFSQ8_HYBRID: { index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFSQ8_HYBRID); } +#endif case IndexType::NSG_MIX: { // TODO(linxj): bug. index = std::make_shared(gpu_device); break; @@ -159,6 +161,8 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { VecIndexPtr LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binary) { auto index = GetVecIndexFactory(index_type); + if (index == nullptr) return nullptr; + // else index->Load(index_binary); return index; } diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 34d3d2f761..2284c44266 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -20,10 +20,10 @@ #include #include -#include "cache/DataObj.h" #include "knowhere/common/BinarySet.h" #include "knowhere/common/Config.h" #include "knowhere/index/vector_index/Quantizer.h" +#include "cache/DataObj.h" #include "utils/Status.h" namespace milvus { From 7bc9039f2c317f18862df308bfaec7e57391bca2 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 11 Oct 2019 19:49:48 +0800 Subject: [PATCH 5/6] MS-635 coding style check Former-commit-id: 06afc01551073cd4ef3c2dc9dace407e441458a7 --- .../knowhere/index/vector_index/IndexIVFSQHybrid.cpp | 9 +-------- cpp/src/core/unittest/test_ivf.cpp | 3 +-- cpp/src/wrapper/VecIndex.cpp | 7 ++++--- cpp/src/wrapper/VecIndex.h | 2 +- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index 5449aec860..5f916f3370 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -231,17 +231,14 @@ IVFSQHybrid::LoadQuantizer(const Config& conf) { void IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { - } void IVFSQHybrid::UnsetQuantizer() { - } void IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { - } IndexModelPtr @@ -260,11 +257,7 @@ IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { } void -IVFSQHybrid::search_impl(int64_t n, - const float* data, - int64_t k, - float* distances, - int64_t* labels, +IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { GPUIVF::search_impl(n, data, k, distances, labels, cfg); } diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index cc4712c263..c6faea9182 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -160,8 +160,7 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, #ifdef CUSTOMIZATION std::make_tuple("IVFSQHybrid", ParameterType::ivfsq), #endif - std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)) - ); + std::make_tuple("GPUIVFSQ", ParameterType::ivfsq))); void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index bd0e8724ad..6bc054bd21 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "wrapper/VecIndex.h" +#include "VecImpl.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" @@ -26,8 +28,6 @@ #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" -#include "wrapper/VecIndex.h" -#include "VecImpl.h" #include "utils/Log.h" #include @@ -161,7 +161,8 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { VecIndexPtr LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binary) { auto index = GetVecIndexFactory(index_type); - if (index == nullptr) return nullptr; + if (index == nullptr) + return nullptr; // else index->Load(index_binary); return index; diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 2284c44266..34d3d2f761 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -20,10 +20,10 @@ #include #include +#include "cache/DataObj.h" #include "knowhere/common/BinarySet.h" #include "knowhere/common/Config.h" #include "knowhere/index/vector_index/Quantizer.h" -#include "cache/DataObj.h" #include "utils/Status.h" namespace milvus { From 762140abc78dfdb9adebb08ab7488b8524381926 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 11 Oct 2019 20:08:05 +0800 Subject: [PATCH 6/6] MS-637 - out of memory when load too many tasks Former-commit-id: 9c40f373286086f57b02da119212b665d1cb8e8c --- cpp/CHANGELOG.md | 1 + cpp/src/scheduler/SchedInst.cpp | 39 ++++------ cpp/src/scheduler/Scheduler.cpp | 7 ++ cpp/src/scheduler/TaskTable.cpp | 12 ++++ cpp/unittest/scheduler/test_normal.cpp | 2 +- cpp/unittest/scheduler/test_resource.cpp | 10 +-- cpp/unittest/scheduler/test_schedinst.cpp | 88 ----------------------- 7 files changed, 40 insertions(+), 119 deletions(-) delete mode 100644 cpp/unittest/scheduler/test_schedinst.cpp diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d0763d5c06..c4015e0231 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-601 - Docker logs error caused by get CPUTemperature error - MS-622 - Delete vectors should be failed if date range is invalid - MS-620 - Get table row counts display wrong error code +- MS-637 - out of memory when load too many tasks ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index cc2b4e280a..194e0d0e00 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -49,40 +49,27 @@ load_simple_config() { std::vector pool; config.GetResourceConfigPool(pool); - bool cpu = false; - std::set gpu_ids; + // get resources + bool use_cpu_to_compute = false; for (auto& resource : pool) { if (resource == "cpu") { - cpu = true; + use_cpu_to_compute = true; break; - } else { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // error - exit(-1); - } - auto gpu_id = std::stoi(resource.substr(3)); - if (gpu_id >= get_num_gpu()) { - // error - exit(-1); - } - gpu_ids.insert(gpu_id); } } + auto gpu_ids = get_gpu_pool(); + // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); - auto io = Connection("io", 500); - if (cpu) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); - ResMgrInst::GetInstance()->Connect("disk", "cpu", io); - } else { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, false)); - ResMgrInst::GetInstance()->Connect("disk", "cpu", io); - auto pcie = Connection("pcie", 12000); - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), io); - } + auto io = Connection("io", 500); + ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute)); + ResMgrInst::GetInstance()->Connect("disk", "cpu", io); + + auto pcie = Connection("pcie", 12000); + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); } } diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index 3a82a1b361..19197b4168 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -110,11 +110,15 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { break; } case TaskLabelType::BROADCAST: { + if (resource->HasExecutor() == false) { + load_completed_event->task_table_item_->Move(); + } Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_->task, resource); break; } default: { break; } } + resource->WakeupLoader(); } } @@ -127,6 +131,9 @@ Scheduler::OnStartUp(const EventPtr& event) { void Scheduler::OnFinishTask(const EventPtr& event) { + if (auto resource = event->resource_.lock()) { + resource->WakeupLoader(); + } } void diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index 0d6742c649..e316c96573 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -17,6 +17,7 @@ #include "scheduler/TaskTable.h" #include "Utils.h" +#include "utils/Log.h" #include "event/TaskTableUpdatedEvent.h" #include @@ -157,6 +158,17 @@ TaskTableItem::Dump() { std::vector TaskTable::PickToLoad(uint64_t limit) { + size_t count = 0; + for (int j = last_finish_ + 1; j < table_.size(); ++j) { + if (not table_[j]) { + SERVER_LOG_WARNING << "table[" << j << "] is nullptr"; + } + if (table_[j]->state == TaskTableItemState::LOADED) { + ++count; + if (count > 2) return std::vector(); + } + } + std::vector indexes; bool cross = false; for (uint64_t i = last_finish_ + 1, count = 0; i < table_.size() && count < limit; ++i) { diff --git a/cpp/unittest/scheduler/test_normal.cpp b/cpp/unittest/scheduler/test_normal.cpp index 1dbd93e044..59c15395e7 100644 --- a/cpp/unittest/scheduler/test_normal.cpp +++ b/cpp/unittest/scheduler/test_normal.cpp @@ -46,7 +46,7 @@ TEST(NormalTest, INST_TEST) { res_mgr->Start(); scheduler->Start(); - const uint64_t NUM_TASK = 1000; + const uint64_t NUM_TASK = 2; std::vector> tasks; ms::TableFileSchemaPtr dummy = nullptr; diff --git a/cpp/unittest/scheduler/test_resource.cpp b/cpp/unittest/scheduler/test_resource.cpp index 31fe425959..9d859d6243 100644 --- a/cpp/unittest/scheduler/test_resource.cpp +++ b/cpp/unittest/scheduler/test_resource.cpp @@ -31,6 +31,8 @@ namespace milvus { namespace scheduler { +constexpr uint64_t max_once_load = 2; + /************ ResourceBaseTest ************/ class ResourceBaseTest : public testing::Test { protected: @@ -182,7 +184,7 @@ class ResourceAdvanceTest : public testing::Test { }; TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { - const uint64_t NUM = 100; + const uint64_t NUM = 10; std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { @@ -208,7 +210,7 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { } TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { - const uint64_t NUM = 100; + const uint64_t NUM = max_once_load; std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { @@ -234,7 +236,7 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { } TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { - const uint64_t NUM = 100; + const uint64_t NUM = max_once_load; std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { @@ -260,7 +262,7 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { } TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { - const uint64_t NUM = 100; + const uint64_t NUM = max_once_load; std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { diff --git a/cpp/unittest/scheduler/test_schedinst.cpp b/cpp/unittest/scheduler/test_schedinst.cpp deleted file mode 100644 index e63a9615bc..0000000000 --- a/cpp/unittest/scheduler/test_schedinst.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include - -#include "server/Config.h" -#include "scheduler/SchedInst.h" - - -namespace milvus { -namespace scheduler { - -class SchedInstTest : public testing::Test { - protected: - void - SetUp() override { - boost::filesystem::create_directory(TMP_DIR); - std::stringstream ss; - ss << "resource_config: " << std::endl; - ss << " resources: " << std::endl; - ss << " ssda: " << std::endl; - ss << " type: DISK" << std::endl; - ss << " device_id: 0" << std::endl; - ss << " enable_loader: true" << std::endl; - ss << " enable_executor: false" << std::endl; - ss << " " << std::endl; - ss << " cpu: " << std::endl; - ss << " type: CPU" << std::endl; - ss << " device_id: 0" << std::endl; - ss << " enable_loader: true" << std::endl; - ss << " enable_executor: false" << std::endl; - ss << " " << std::endl; - ss << " gpu0: " << std::endl; - ss << " type: GPU" << std::endl; - ss << " device_id: 0" << std::endl; - ss << " enable_loader: true" << std::endl; - ss << " enable_executor: true" << std::endl; - ss << " " << std::endl; - ss << " connections: " << std::endl; - ss << " io: " << std::endl; - ss << " speed: 500" << std::endl; - ss << " endpoint: ssda===cpu" << std::endl; - ss << " pcie: " << std::endl; - ss << " speed: 11000" << std::endl; - ss << " endpoint: cpu===gpu0" << std::endl; - - boost::filesystem::path fpath(CONFIG_FILE); - boost::filesystem::fstream fstream(fpath, std::ios_base::out); - fstream << ss.str(); - fstream.close(); - - server::Config::GetInstance().LoadConfigFile(CONFIG_FILE); - } - - void - TearDown() override { - StopSchedulerService(); - boost::filesystem::remove_all(TMP_DIR); - } - - const std::string TMP_DIR = "/tmp/milvus_sched_test"; - const std::string CONFIG_FILE = "/tmp/milvus_sched_test/config.yaml"; -}; - -TEST_F(SchedInstTest, SIMPLE_GPU) { - StartSchedulerService(); -} - -} // namespace scheduler -} // namespace milvus - - -