From f136df4aaa29075b6efa82b17834fdd46cd0888a Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 12 Sep 2019 14:10:56 +0800 Subject: [PATCH] MS-543 send error status to sdk when load index fail Former-commit-id: a62922b904c7f7d478f5211b4d215d387ea23ae2 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 3 +++ cpp/src/db/Status.cpp | 2 +- cpp/src/db/scheduler/context/SearchContext.h | 5 ++++- cpp/src/scheduler/task/SearchTask.cpp | 17 +++++++++++++---- cpp/src/utils/Error.h | 1 + 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index abadec93fa..7a9482a233 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -34,6 +34,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-474 - Create index hang if use branch-0.3.1 server config - MS-510 - unittest out of memory and crashed - MS-507 - Dataset 10m-512, index type sq8，performance in-normal when set CPU_CACHE to 16 or 64 +- MS-543 - SearchTask fail without exception ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 2924d1e718..80b1994778 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -424,6 +424,9 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch scheduler.Schedule(context); context->WaitResult(); + if (!context->GetStatus().ok()) { + return context->GetStatus(); + } //step 3: print time cost information double load_cost = context->LoadCost(); diff --git a/cpp/src/db/Status.cpp b/cpp/src/db/Status.cpp index 5744f1bdbe..d19b1e66ff 100644 --- a/cpp/src/db/Status.cpp +++ b/cpp/src/db/Status.cpp @@ -69,7 +69,7 @@ std::string Status::ToString() const { type = "InvalidPath: "; break; default: - snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", + snprintf(tmp, sizeof(tmp), "Error code(0x%x): ", static_cast(code())); type = tmp; break; diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index 9ca03e0830..a950962b6e 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -38,7 +38,9 @@ public: const ResultSet& GetResult() const { return result_; } ResultSet& GetResult() { return result_; } - std::string Identity() const { return identity_; } + const std::string& Identity() const { return identity_; } + const Status& GetStatus() const { return status_; } + Status& GetStatus() { return status_; } void IndexSearchDone(size_t index_id); void WaitResult(); @@ -64,6 +66,7 @@ private: std::condition_variable done_cond_; std::string identity_; //for debug + Status status_; double time_cost_load_ = 0.0; //time cost for load all index files, unit: us double time_cost_search_ = 0.0; //time cost for entire search, unit: us diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 59f6509c02..937936dd31 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -98,14 +98,18 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { server::TimeRecorder rc(""); Status stat = Status::OK(); std::string error_msg; + std::string type_str; try { if (type == LoadType::DISK2CPU) { stat = index_engine_->Load(); + type_str = "DISK2CPU"; } else if (type == LoadType::CPU2GPU) { stat = index_engine_->CopyToGpu(device_id); + type_str = "CPU2GPU"; } else if (type == LoadType::GPU2CPU) { stat = index_engine_->CopyToCpu(); + type_str = "GPU2CPU"; } else { error_msg = "Wrong load type"; stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); @@ -117,13 +121,18 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { } if (!stat.ok()) { - if (error_msg.empty()) - error_msg = std::string("Failed to load index file: file not available"); - //typical error: file not available - ENGINE_LOG_ERROR << error_msg; + Status s; + if (stat.ToString().find("out of memory") != std::string::npos) { + error_msg = "out of memory: " + type_str; + s = Status(SERVER_OUT_OF_MEMORY, error_msg); + } else { + error_msg = "Failed to load index file: " + type_str; + s = Status(SERVER_UNEXPECTED_ERROR, error_msg); + } for (auto &context : search_contexts_) { context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed + context->GetStatus() = s; } return; diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 4ee2da8d5f..f49e4b36af 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -66,6 +66,7 @@ constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); constexpr ErrorCode SERVER_INVALID_INDEX_FILE_SIZE = ToServerErrorCode(116); +constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117); //db error code constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);