From ded33555386c8c474cf5aa23f5b80fe5d0d605cf Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 10 Sep 2019 19:31:29 +0800 Subject: [PATCH 1/7] MS-539 Remove old task code Former-commit-id: 89d66627b558cb33e5b482520e256b701e5d7868 --- cpp/CHANGELOG.md | 1 + cpp/src/db/scheduler/TaskScheduler.cpp | 12 - cpp/src/db/scheduler/task/DeleteTask.cpp | 5 - cpp/src/db/scheduler/task/IndexLoadTask.cpp | 71 +----- cpp/src/db/scheduler/task/SearchTask.cpp | 249 +------------------- cpp/src/db/scheduler/task/SearchTask.h | 16 -- cpp/unittest/db/search_test.cpp | 38 +-- 7 files changed, 23 insertions(+), 369 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index aff760ebec..1bdfef13b2 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -113,6 +113,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-531 - Disable next version code - MS-533 - Update resource_test to cover dump function - MS-523 - Config file validation +- MS-539 - Remove old task code ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index 0a50c02bfa..3e1807b8c5 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -89,7 +89,6 @@ TaskScheduler::TaskDispatchWorker() { return true; } -#ifdef NEW_SCHEDULER // TODO: Put task into Disk-TaskTable auto task = TaskConvert(task_ptr); auto disk_list = ResMgrInst::GetInstance()->GetDiskResources(); @@ -98,16 +97,7 @@ TaskScheduler::TaskDispatchWorker() { disk->task_table().Put(task); } } -#else - //execute task - ScheduleTaskPtr next_task = task_ptr->Execute(); - if(next_task != nullptr) { - task_queue_.Put(next_task); - } -#endif } - - return true; } bool @@ -126,8 +116,6 @@ TaskScheduler::TaskWorker() { task_queue_.Put(next_task); } } - - return true; } } diff --git a/cpp/src/db/scheduler/task/DeleteTask.cpp b/cpp/src/db/scheduler/task/DeleteTask.cpp index a4660fc157..975a595e6f 100644 --- a/cpp/src/db/scheduler/task/DeleteTask.cpp +++ b/cpp/src/db/scheduler/task/DeleteTask.cpp @@ -17,11 +17,6 @@ DeleteTask::DeleteTask(const DeleteContextPtr& context) } std::shared_ptr DeleteTask::Execute() { - - if(context_ != nullptr && context_->meta() != nullptr) { - context_->meta()->DeleteTableFiles(context_->table_id()); - } - return nullptr; } diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 65bde80237..4efb041d61 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -15,82 +15,13 @@ namespace zilliz { namespace milvus { namespace engine { -namespace { -void CollectFileMetrics(int file_type, size_t file_size) { - switch(file_type) { - case meta::TableFileSchema::RAW: - case meta::TableFileSchema::TO_INDEX: { - server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); - server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); - server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); - break; - } - default: { - server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size); - server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size); - server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size); - break; - } - } -} -} - IndexLoadTask::IndexLoadTask() : IScheduleTask(ScheduleTaskType::kIndexLoad) { } std::shared_ptr IndexLoadTask::Execute() { - server::TimeRecorder rc(""); - //step 1: load index - ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_, - file_->location_, - (EngineType)file_->engine_type_, - (MetricType)file_->metric_type_, - file_->nlist_); - - try { - auto stat = index_ptr->Load(); - if(!stat.ok()) { - //typical error: file not available - ENGINE_LOG_ERROR << "Failed to load index file: file not available"; - - for(auto& context : search_contexts_) { - context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed - } - - return nullptr; - } - } catch (std::exception& ex) { - //typical error: out of disk space or permition denied - std::string msg = "Failed to load index file: " + std::string(ex.what()); - ENGINE_LOG_ERROR << msg; - - for(auto& context : search_contexts_) { - context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed - } - - return nullptr; - } - - size_t file_size = index_ptr->PhysicalSize(); - - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + std::to_string(file_->file_type_) - + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; - double span = rc.ElapseFromBegin(info); - for(auto& context : search_contexts_) { - context->AccumLoadCost(span); - } - - CollectFileMetrics(file_->file_type_, file_size); - - //step 2: return search task for later execution - SearchTaskPtr task_ptr = std::make_shared(); - task_ptr->index_id_ = file_->id_; - task_ptr->file_type_ = file_->file_type_; - task_ptr->index_engine_ = index_ptr; - task_ptr->search_contexts_.swap(search_contexts_); - return std::static_pointer_cast(task_ptr); + return nullptr; } } diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index fffd582112..5117dc948c 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -14,259 +14,12 @@ namespace zilliz { namespace milvus { namespace engine { -namespace { - -static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 1000000; -static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; - -bool NeedParallelReduce(uint64_t nq, uint64_t topk) { - server::ServerConfig &config = server::ServerConfig::GetInstance(); - server::ConfigNode& db_config = config.GetConfig(server::CONFIG_DB); - bool need_parallel = db_config.GetBoolValue(server::CONFIG_DB_PARALLEL_REDUCE, false); - if(!need_parallel) { - return false; - } - - return nq*topk >= PARALLEL_REDUCE_THRESHOLD; -} - -void ParallelReduce(std::function& reduce_function, size_t max_index) { - size_t reduce_batch = PARALLEL_REDUCE_BATCH; - - auto thread_count = std::thread::hardware_concurrency() - 1; //not all core do this work - if(thread_count > 0) { - reduce_batch = max_index/thread_count + 1; - } - ENGINE_LOG_DEBUG << "use " << thread_count << - " thread parallelly do reduce, each thread process " << reduce_batch << " vectors"; - - std::vector > thread_array; - size_t from_index = 0; - while(from_index < max_index) { - size_t to_index = from_index + reduce_batch; - if(to_index > max_index) { - to_index = max_index; - } - - auto reduce_thread = std::make_shared(reduce_function, from_index, to_index); - thread_array.push_back(reduce_thread); - - from_index = to_index; - } - - for(auto& thread_ptr : thread_array) { - thread_ptr->join(); - } -} - -} - SearchTask::SearchTask() : IScheduleTask(ScheduleTaskType::kSearch) { } std::shared_ptr SearchTask::Execute() { - if(index_engine_ == nullptr) { - return nullptr; - } - - ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_<< " with " - << search_contexts_.size() << " tasks"; - - server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); - - server::CollectSearchTaskMetrics metrics(file_type_); - - bool metric_l2 = (index_engine_->IndexMetricType() == MetricType::L2); - - std::vector output_ids; - std::vector output_distence; - for(auto& context : search_contexts_) { - //step 1: allocate memory - auto inner_k = context->topk(); - auto nprobe = context->nprobe(); - output_ids.resize(inner_k*context->nq()); - output_distence.resize(inner_k*context->nq()); - - try { - //step 2: search - index_engine_->Search(context->nq(), context->vectors(), inner_k, nprobe, output_distence.data(), - output_ids.data()); - - double span = rc.RecordSection("do search for context:" + context->Identity()); - context->AccumSearchCost(span); - - //step 3: cluster result - SearchContext::ResultSet result_set; - auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); - SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); - - span = rc.RecordSection("cluster result for context:" + context->Identity()); - context->AccumReduceCost(span); - - //step 4: pick up topk result - SearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult()); - - span = rc.RecordSection("reduce topk for context:" + context->Identity()); - context->AccumReduceCost(span); - - } catch (std::exception& ex) { - ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); - context->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed - continue; - } - - //step 5: notify to send result to client - context->IndexSearchDone(index_id_); - } - - rc.ElapseFromBegin("totally cost"); - - return nullptr; -} - -Status SearchTask::ClusterResult(const std::vector &output_ids, - const std::vector &output_distence, - uint64_t nq, - uint64_t topk, - SearchContext::ResultSet &result_set) { - if(output_ids.size() < nq*topk || output_distence.size() < nq*topk) { - std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + - " distance array size: " + std::to_string(output_distence.size()); - ENGINE_LOG_ERROR << msg; - return Status(DB_ERROR, msg); - } - - result_set.clear(); - result_set.resize(nq); - - std::function reduce_worker = [&](size_t from_index, size_t to_index) { - for (auto i = from_index; i < to_index; i++) { - SearchContext::Id2DistanceMap id_distance; - id_distance.reserve(topk); - for (auto k = 0; k < topk; k++) { - uint64_t index = i * topk + k; - if(output_ids[index] < 0) { - continue; - } - id_distance.push_back(std::make_pair(output_ids[index], output_distence[index])); - } - result_set[i] = id_distance; - } - }; - - if(NeedParallelReduce(nq, topk)) { - ParallelReduce(reduce_worker, nq); - } else { - reduce_worker(0, nq); - } - - return Status::OK(); -} - -Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, - SearchContext::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending) { - //Note: the score_src and score_target are already arranged by score in ascending order - if(distance_src.empty()) { - ENGINE_LOG_WARNING << "Empty distance source array"; - return Status::OK(); - } - - if(distance_target.empty()) { - distance_target.swap(distance_src); - return Status::OK(); - } - - size_t src_count = distance_src.size(); - size_t target_count = distance_target.size(); - SearchContext::Id2DistanceMap distance_merged; - distance_merged.reserve(topk); - size_t src_index = 0, target_index = 0; - while(true) { - //all score_src items are merged, if score_merged.size() still less than topk - //move items from score_target to score_merged until score_merged.size() equal topk - if(src_index >= src_count) { - for(size_t i = target_index; i < target_count && distance_merged.size() < topk; ++i) { - distance_merged.push_back(distance_target[i]); - } - break; - } - - //all score_target items are merged, if score_merged.size() still less than topk - //move items from score_src to score_merged until score_merged.size() equal topk - if(target_index >= target_count) { - for(size_t i = src_index; i < src_count && distance_merged.size() < topk; ++i) { - distance_merged.push_back(distance_src[i]); - } - break; - } - - //compare score, - // if ascending = true, put smallest score to score_merged one by one - // else, put largest score to score_merged one by one - auto& src_pair = distance_src[src_index]; - auto& target_pair = distance_target[target_index]; - if(ascending){ - if(src_pair.second > target_pair.second) { - distance_merged.push_back(target_pair); - target_index++; - } else { - distance_merged.push_back(src_pair); - src_index++; - } - } else { - if(src_pair.second < target_pair.second) { - distance_merged.push_back(target_pair); - target_index++; - } else { - distance_merged.push_back(src_pair); - src_index++; - } - } - - //score_merged.size() already equal topk - if(distance_merged.size() >= topk) { - break; - } - } - - distance_target.swap(distance_merged); - - return Status::OK(); -} - -Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, - uint64_t topk, - bool ascending, - SearchContext::ResultSet &result_target) { - if (result_target.empty()) { - result_target.swap(result_src); - return Status::OK(); - } - - if (result_src.size() != result_target.size()) { - std::string msg = "Invalid result set size"; - ENGINE_LOG_ERROR << msg; - return Status(DB_ERROR, msg); - } - - std::function ReduceWorker = [&](size_t from_index, size_t to_index) { - for (size_t i = from_index; i < to_index; i++) { - SearchContext::Id2DistanceMap &score_src = result_src[i]; - SearchContext::Id2DistanceMap &score_target = result_target[i]; - SearchTask::MergeResult(score_src, score_target, topk, ascending); - } - }; - - if(NeedParallelReduce(result_src.size(), topk)) { - ParallelReduce(ReduceWorker, result_src.size()); - } else { - ReduceWorker(0, result_src.size()); - } - - return Status::OK(); + return nullptr; } } diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index 6010046446..88a51c2c84 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -19,22 +19,6 @@ public: virtual std::shared_ptr Execute() override; - static Status ClusterResult(const std::vector &output_ids, - const std::vector &output_distence, - uint64_t nq, - uint64_t topk, - SearchContext::ResultSet &result_set); - - static Status MergeResult(SearchContext::Id2DistanceMap &distance_src, - SearchContext::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending); - - static Status TopkResult(SearchContext::ResultSet &result_src, - uint64_t topk, - bool ascending, - SearchContext::ResultSet &result_target); - public: size_t index_id_ = 0; int file_type_ = 0; //for metrics diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index cd2ff3b5c6..b8e0ffafff 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -10,6 +10,8 @@ #include #include #include +#include + using namespace zilliz::milvus; @@ -114,23 +116,23 @@ TEST(DBSearchTest, TOPK_TEST) { std::vector target_ids; std::vector target_distence; engine::SearchContext::ResultSet src_result; - auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_FALSE(status.ok()); ASSERT_TRUE(src_result.empty()); BuildResult(NQ, TOP_K, ascending, target_ids, target_distence); - status = engine::SearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + status = engine::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), NQ); engine::SearchContext::ResultSet target_result; - status = engine::SearchTask::TopkResult(target_result, TOP_K, ascending, target_result); + status = engine::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(target_result, TOP_K, ascending, src_result); + status = engine::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result); ASSERT_FALSE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); ASSERT_TRUE(src_result.empty()); ASSERT_EQ(target_result.size(), NQ); @@ -140,10 +142,10 @@ TEST(DBSearchTest, TOPK_TEST) { uint64_t wrong_topk = TOP_K - 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = engine::SearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); + status = engine::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -152,7 +154,7 @@ TEST(DBSearchTest, TOPK_TEST) { wrong_topk = TOP_K + 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -170,15 +172,15 @@ TEST(DBSearchTest, MERGE_TEST) { uint64_t src_count = 5, target_count = 8; BuildResult(1, src_count, ascending, src_ids, src_distence); BuildResult(1, target_count, ascending, target_ids, target_distence); - auto status = engine::SearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); + auto status = engine::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); + status = engine::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); ASSERT_TRUE(status.ok()); { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 10, ascending); + status = engine::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); CheckResult(src_result[0], target_result[0], target, ascending); @@ -187,7 +189,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target; - status = engine::SearchTask::MergeResult(src, target, 10, ascending); + status = engine::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); ASSERT_TRUE(src.empty()); @@ -197,7 +199,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30, ascending); + status = engine::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); @@ -206,7 +208,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap target = src_result[0]; engine::SearchContext::Id2DistanceMap src = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30, ascending); + status = engine::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); @@ -229,7 +231,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { BuildResult(nq, topk, ascending, target_ids, target_distence); rc.RecordSection("build id/dietance map"); - auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), nq); @@ -269,14 +271,14 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) { server::TimeRecorder rc("DoCluster"); BuildResult(nq, topk, ascending, target_ids, target_distence); - auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); rc.RecordSection("cluster result"); BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence); - status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); + status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); rc.RecordSection("cluster result"); - engine::SearchTask::TopkResult(insufficient_result, topk, ascending, src_result); + engine::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result); ASSERT_TRUE(status.ok()); rc.RecordSection("topk"); From c811c3a781dd679afeca50dee7696875d5d1beb7 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 10 Sep 2019 19:42:18 +0800 Subject: [PATCH 2/7] update mysql meta test Former-commit-id: 5f6d059f81b16ae34354324c16f342c3125c8b92 --- cpp/src/db/meta/MySQLMetaImpl.h | 2 +- cpp/unittest/db/mysql_meta_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 64c703b688..97604ef3ea 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -53,7 +53,7 @@ class MySQLMetaImpl : public Meta { Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag); + Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index e7529506b0..ede879c3ba 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -351,7 +351,7 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { status = impl_->DeleteTable(table_id); ASSERT_TRUE(status.ok()); - status = impl_->CleanUpFilesWithTTL(1UL); + status = impl_->CleanUpFilesWithTTL(0UL); ASSERT_TRUE(status.ok()); } From 25210886d02b743189eeb867310836cd036b091e Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 10 Sep 2019 19:59:50 +0800 Subject: [PATCH 3/7] add unitest Former-commit-id: 4843e1b14fa97b253a155a71ddb02f7c076942c4 --- cpp/unittest/db/meta_tests.cpp | 6 +++ cpp/unittest/db/misc_test.cpp | 10 ++++- cpp/unittest/server/config_test.cpp | 3 +- cpp/unittest/server/util_test.cpp | 59 ++++++++++++++++++++++------- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 51a6f8e223..60b6eab310 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -324,6 +324,12 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { status = impl_->DeleteTableFiles(table_id); ASSERT_TRUE(status.ok()); + status = impl_->CreateTableFile(table_file); + table_file.file_type_ = meta::TableFileSchema::NEW; + status = impl_->UpdateTableFile(table_file); + status = impl_->CleanUp(); + ASSERT_TRUE(status.ok()); + status = impl_->DeleteTable(table_id); ASSERT_TRUE(status.ok()); diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index bc853c8f23..684694a3c4 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -128,6 +128,14 @@ TEST(DBMiscTest, UTILS_TEST) { ASSERT_TRUE(boost::filesystem::exists(path)); } + options.slave_paths.push_back("/"); + status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_FALSE(status.ok()); + + options.path = "/"; + status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_FALSE(status.ok()); + engine::meta::TableFileSchema file; file.id_ = 50; file.table_id_ = TABLE_NAME; @@ -142,6 +150,4 @@ TEST(DBMiscTest, UTILS_TEST) { status = engine::utils::DeleteTableFilePath(options, file); ASSERT_TRUE(status.ok()); - - } \ No newline at end of file diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index e23f085059..d512cd82bc 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -101,7 +101,8 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { err = server::ServerConfig::GetInstance().ValidateConfig(); ASSERT_EQ(err, SERVER_SUCCESS); - server::ConfigNode node1 = config.GetConfig("server_config"); + const server::ServerConfig& config_const = config; + server::ConfigNode node1 = config_const.GetConfig("server_config"); server::ConfigNode& node2 = config.GetConfig("cache_config"); node1.Combine(node2); diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 730eedbd33..f51287ea4f 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -158,7 +158,17 @@ TEST(UtilTest, LOG_TEST) { ASSERT_EQ(fname, "log_config.conf"); } -TEST(UtilTest, VALIDATE_TABLENAME_TEST) { +TEST(UtilTest, TIMERECORDER_TEST) { + for(int64_t log_level = 0; log_level <= 6; log_level++) { + if(log_level == 5) { + continue; //skip fatal + } + server::TimeRecorder rc("time", log_level); + rc.RecordSection("end"); + } +} + +TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; ErrorCode res = server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_SUCCESS); @@ -192,7 +202,7 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) { ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); } -TEST(UtilTest, VALIDATE_DIMENSIONTEST) { +TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); @@ -200,7 +210,7 @@ TEST(UtilTest, VALIDATE_DIMENSIONTEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); } -TEST(UtilTest, VALIDATE_INDEX_TEST) { +TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); @@ -218,14 +228,14 @@ TEST(UtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), SERVER_SUCCESS); } -TEST(ValidationUtilTest, ValidateTopkTest) { +TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { engine::meta::TableSchema schema; ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), SERVER_SUCCESS); } -TEST(ValidationUtilTest, ValidateNprobeTest) { +TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) { engine::meta::TableSchema schema; schema.nlist_ = 100; ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), SERVER_SUCCESS); @@ -233,7 +243,7 @@ TEST(ValidationUtilTest, ValidateNprobeTest) { ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), SERVER_SUCCESS); } -TEST(ValidationUtilTest, ValidateGpuTest) { +TEST(ValidationUtilTest, VALIDATE_GPU_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), SERVER_SUCCESS); @@ -242,12 +252,33 @@ TEST(ValidationUtilTest, ValidateGpuTest) { ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), SERVER_SUCCESS); } -TEST(UtilTest, TIMERECORDER_TEST) { - for(int64_t log_level = 0; log_level <= 6; log_level++) { - if(log_level == 5) { - continue; //skip fatal - } - server::TimeRecorder rc("time", log_level); - rc.RecordSection("end"); - } +TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) { + ASSERT_EQ(server::ValidationUtil::ValidateIpAddress("127.0.0.1"), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateIpAddress("not ip"), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) { + ASSERT_EQ(server::ValidationUtil::ValidateStringIsNumber("1234"), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsNumber("not number"), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) { + std::string str = "true"; + ASSERT_EQ(server::ValidationUtil::ValidateStringIsBool(str), SERVER_SUCCESS); + str = "not bool"; + ASSERT_NE(server::ValidationUtil::ValidateStringIsBool(str), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) { + double ret = 0.0; + ASSERT_EQ(server::ValidationUtil::ValidateStringIsDouble("2.5", ret), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsDouble("not double", ret), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) { + ASSERT_EQ(server::ValidationUtil::ValidateDbURI("sqlite://:@:/"), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("xxx://:@:/"), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("not uri"), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus"), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus"), SERVER_SUCCESS); } From 9665820452ba390a46252a1e522bd992c8c0eac4 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 10 Sep 2019 20:09:56 +0800 Subject: [PATCH 4/7] add unitest Former-commit-id: baa39b21f9095d00ef9c6ee43027d5acd993c54f --- cpp/unittest/db/mem_test.cpp | 21 ++------------------- cpp/unittest/db/utils.h | 2 +- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 460003a9f1..a612be2bb3 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -55,9 +55,6 @@ void BuildVectors(int64_t n, std::vector &vectors) { } TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { - - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); - engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); @@ -96,16 +93,10 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { vector_ids = source.GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); - - - status = impl_->DropAll(); - ASSERT_TRUE(status.ok()); } TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { - - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); - auto options = engine::OptionsFactory::Build(); + auto options = GetOptions(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); @@ -143,15 +134,10 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { ASSERT_EQ(vector_ids.size(), n_max - n_100); ASSERT_TRUE(mem_table_file.IsFull()); - - status = impl_->DropAll(); - ASSERT_TRUE(status.ok()); } TEST_F(MemManagerTest, MEM_TABLE_TEST) { - - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); - auto options = engine::OptionsFactory::Build(); + auto options = GetOptions(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); @@ -211,9 +197,6 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { status = mem_table.Serialize(); ASSERT_TRUE(status.ok()); - - status = impl_->DropAll(); - ASSERT_TRUE(status.ok()); } TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 8be16dd46c..8cb006ecfc 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -87,7 +87,7 @@ class MySqlMetaTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class MemManagerTest : public BaseTest { +class MemManagerTest : public MetaTest { }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 5ca56a1718011f03eb6d516b3912b938fc00f0d2 Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Tue, 10 Sep 2019 20:14:58 +0800 Subject: [PATCH 5/7] MS-311 1. error log remove compile address Former-commit-id: ff24292efc9765678f880362edffe180d8fb8192 --- cpp/src/core/src/knowhere/common/exception.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cpp/src/core/src/knowhere/common/exception.cpp b/cpp/src/core/src/knowhere/common/exception.cpp index aec8c03650..1697f14910 100644 --- a/cpp/src/core/src/knowhere/common/exception.cpp +++ b/cpp/src/core/src/knowhere/common/exception.cpp @@ -14,11 +14,23 @@ namespace knowhere { KnowhereException::KnowhereException(const std::string &msg):msg(msg) {} KnowhereException::KnowhereException(const std::string &m, const char *funcName, const char *file, int line) { +#ifdef DEBUG int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", funcName, file, line, m.c_str()); msg.resize(size + 1); snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", funcName, file, line, m.c_str()); +#else + std::string file_path(file); + auto const pos = file_path.find_last_of('/'); + auto filename = file_path.substr(pos+1).c_str(); + + int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", + funcName, filename, line, m.c_str()); + msg.resize(size + 1); + snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", + funcName, filename, line, m.c_str()); +#endif } const char *KnowhereException::what() const noexcept { From b76ad6b073697d4c968694316212cd94ca46a909 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 11 Sep 2019 10:17:26 +0800 Subject: [PATCH 6/7] update server config Former-commit-id: 540f2d12259f922f7eef41e6cda5a29ec19b0197 --- cpp/conf/server_config.template | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index ea41fd144f..be4d4f5430 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -34,7 +34,6 @@ cache_config: gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 gpu_ids: # gpu id - 0 - - 1 engine_config: use_blas_threshold: 20 From 563baa52e88566b424550fd79d4656824a0b2510 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 11 Sep 2019 10:31:19 +0800 Subject: [PATCH 7/7] fix unitest failure Former-commit-id: 8582ebb9772270e41e74e4c582044e6e8b732d77 --- cpp/unittest/server/appendix/server_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index 35e4477d86..b0095b8d37 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -34,7 +34,6 @@ cache_config: gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 gpu_ids: # gpu id - 0 - - 1 engine_config: use_blas_threshold: 20