From ce7e835658a331c51bed12a6f5b3bd4521d54419 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 9 Aug 2019 15:34:11 +0800 Subject: [PATCH 1/3] handle exception in merge file Former-commit-id: 0b1503599944acafa30e12642c3e00c915afc223 --- cpp/src/db/DBImpl.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index d8f894fbc8..dd56f79755 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -363,6 +363,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { ENGINE_LOG_DEBUG << "Merge files for table " << table_id; + //step 1: create table file meta::TableFileSchema table_file; table_file.table_id_ = table_id; table_file.date_ = date; @@ -374,6 +375,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, return status; } + //step 2: merge files ExecutionEnginePtr index = EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_); @@ -397,9 +399,25 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, if (index_size >= options_.index_trigger_size) break; } + //step 3: serialize to disk + try { + index->Serialize(); + } catch (std::exception& ex) { + //typical error: out of disk space or permition denied + std::string msg = "Serialize merged index encounter exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; - index->Serialize(); + table_file.file_type_ = meta::TableFileSchema::TO_DELETE; + status = meta_ptr_->UpdateTableFile(table_file); + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + std::cout << "ERROR: failed to persist merged index file: " << table_file.location_ + << ", possible out of disk space" << std::endl; + + return Status::Error(msg); + } + + //step 4: update table files state if (index_size >= options_.index_trigger_size) { table_file.file_type_ = meta::TableFileSchema::TO_INDEX; } else { From 414be9021d1e116c62e4bebc3ebdbc1c45515aaa Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 9 Aug 2019 18:19:07 +0800 Subject: [PATCH 2/3] MS-330 Stability test failed caused by server core dumped Former-commit-id: 66079f9b51b0e40adfc6dfbcf2d2156431124ebb --- cpp/CHANGELOG.md | 1 + cpp/src/db/Utils.cpp | 4 +++- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index cf73d770e2..c18fd2b718 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-235 - Some test cases random fail - MS-236 - Add MySQLMetaImpl::HasNonIndexFiles - MS-257 - Update bzip2 download url +- MS-330 - Stability test failed caused by server core dumped ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 933ca06bd5..98a4f356df 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -131,7 +131,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab } } - return Status::Error("Table file doesn't exist: " + table_file.file_id_); + std::string msg = "Table file doesn't exist: " + table_file.file_id_; + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index f3aacf8352..d33828391a 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -46,7 +46,20 @@ std::shared_ptr IndexLoadTask::Execute() { ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType)file_->engine_type_); - index_ptr->Load(); + + try { + index_ptr->Load(); + } catch (std::exception& ex) { + //typical error: out of disk space or permition denied + std::string msg = "Failed to load index file: " + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + + for(auto& context : search_contexts_) { + context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed + } + + return nullptr; + } size_t file_size = index_ptr->PhysicalSize(); From 039a268f22f066de4a30eb5b2f5ee9890eb8807c Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 12 Aug 2019 14:06:08 +0800 Subject: [PATCH 3/3] MS-330 Stability test failed caused by server core dumped Former-commit-id: 6399857193109ff8d5886c9b8db168cda04026bb --- cpp/src/db/DBImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index dd56f79755..8ff2562a72 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -482,7 +482,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 1; + int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; }