From f00ca24f1b5ae4f7a32bfde28aad74a37c817bf4 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 14:25:02 +0800 Subject: [PATCH 1/7] Disable cleanup if mode is read only Former-commit-id: 09655e2bb8a52466c1732c829519742867cf07a3 --- cpp/CHANGELOG.md | 1 + cpp/src/db/MySQLMetaImpl.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ac3b2c51f6..19fbbe9c0c 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size +- MS-148 - Disable cleanup if mode is read only ## Improvement - MS-82 - Update server startup welcome message diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 92bb17168a..20b0cef0c6 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -162,7 +162,9 @@ namespace meta { ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); try { - CleanUp(); + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); + } { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); @@ -457,7 +459,7 @@ namespace meta { } //Scoped Connection - if (mode_ != Options::MODE::SINGLE) { + if (mode_ == Options::MODE::CLUSTER) { DeleteTableFiles(table_id); } From cba93cf25db6ecda1b527d0f2b023f32dec5b078 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 14:27:19 +0800 Subject: [PATCH 2/7] Disable cleanup if mode is read only Former-commit-id: 8e0119a612c7da87f77c27453144d6666896bdf9 --- cpp/src/db/MySQLMetaImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 20b0cef0c6..f2e032dac5 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1814,7 +1814,9 @@ namespace meta { MySQLMetaImpl::~MySQLMetaImpl() { // std::lock_guard lock(mysql_mutex); - CleanUp(); + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); + } } } // namespace meta From e89f9b18c10f5ee0460ffc2e7b192a2bcce44b91 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 15:48:21 +0800 Subject: [PATCH 3/7] update Former-commit-id: b8255e6455c40e0ee843a56787502b65312b2a4a --- cpp/src/server/RequestTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 1b91883af5..925bc601cc 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,6 +482,7 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); + SERVER_LOG_DEBUG << file_id_array_ << std::endl; if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 0b003144a5f34296701db804a3586ac99839c446 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 16:04:00 +0800 Subject: [PATCH 4/7] update Former-commit-id: b75d6fad2d4165cc08ea1b528a9cff36791344b1 --- cpp/src/server/RequestTask.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 925bc601cc..d2c2474999 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,7 +482,11 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << file_id_array_ << std::endl; + SERVER_LOG_DEBUG << "file_id_array_: "; + for (auto& file_id : file_id_array_) { + SERVER_LOG_DEBUG << file_id; + } + if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From fd6d77d62716e771c64b6f0e1eb0b7a3a7fdfed2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 17:24:36 +0800 Subject: [PATCH 5/7] update Former-commit-id: eb53fcbf6029d7c061d5c87835700b14a1213e63 --- cpp/src/db/DBImpl.cpp | 4 ++++ cpp/src/db/MySQLMetaImpl.cpp | 10 ++++++---- cpp/src/server/RequestHandler.cpp | 2 ++ cpp/src/server/RequestTask.cpp | 5 ----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 0a1e8651e1..352fcae7d0 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -191,6 +191,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } + for (auto& file_schema : files_array) { + ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; + } + if(files_array.empty()) { return Status::Error("Invalid file id"); } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index f2e032dac5..5bef070337 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1083,10 +1083,10 @@ namespace meta { // } Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); @@ -1106,6 +1106,8 @@ namespace meta { TableFileSchema file_schema; + file_schema.id_ = resRow["id"]; + file_schema.table_id_ = table_id; file_schema.engine_type_ = resRow["engine_type"]; diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 037f80e0db..a4dc182c35 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -53,6 +53,7 @@ RequestHandler::SearchVector(std::vector &_return, const std::vector &query_record_array, const std::vector &query_range_array, const int64_t topk) { +// SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVector"; BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, std::vector(), query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); @@ -65,6 +66,7 @@ RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResul const std::vector<::milvus::thrift::RowRecord> &query_record_array, const std::vector<::milvus::thrift::Range> &query_range_array, const int64_t topk) { +// SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVectorInFiles. file_id_array size = " << std::to_string(file_id_array.size()); BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, file_id_array, query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index d2c2474999..1b91883af5 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,11 +482,6 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << "file_id_array_: "; - for (auto& file_id : file_id_array_) { - SERVER_LOG_DEBUG << file_id; - } - if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From ff9a3140c5bf08a8db78058fe9987eee8fed969d Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:31:45 +0800 Subject: [PATCH 6/7] update Former-commit-id: 2543b4f174931936a829eba157aec1fab6c0ef84 --- cpp/src/db/DBImpl.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 352fcae7d0..0a1e8651e1 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -191,10 +191,6 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } - for (auto& file_schema : files_array) { - ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; - } - if(files_array.empty()) { return Status::Error("Invalid file id"); } From 2586e4bf6e74153d83a079e19f2e77b9fe476842 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:44:29 +0800 Subject: [PATCH 7/7] update Former-commit-id: 68476662b5f24c76c79ec63f6a9dd556e3008f74 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 8d7186911b..630b86d735 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size - MS-148 - Disable cleanup if mode is read only +- MS-149 - Fixed searching only one index file issue in distributed mode ## Improvement - MS-82 - Update server startup welcome message