diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index b7b360b30f..26b0226949 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -33,6 +33,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-236 - Add MySQLMetaImpl::HasNonIndexFiles - MS-257 - Update bzip2 download url - MS-288 - Update compile scripts +- MS-330 - Stability test failed caused by server core dumped ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 5df588bda4..636efc51bd 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -370,6 +370,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { ENGINE_LOG_DEBUG << "Merge files for table " << table_id; + //step 1: create table file meta::TableFileSchema table_file; table_file.table_id_ = table_id; table_file.date_ = date; @@ -381,6 +382,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, return status; } + //step 2: merge files ExecutionEnginePtr index = EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_); @@ -404,9 +406,25 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, if (index_size >= options_.index_trigger_size) break; } + //step 3: serialize to disk + try { + index->Serialize(); + } catch (std::exception& ex) { + //typical error: out of disk space or permition denied + std::string msg = "Serialize merged index encounter exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; - index->Serialize(); + table_file.file_type_ = meta::TableFileSchema::TO_DELETE; + status = meta_ptr_->UpdateTableFile(table_file); + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + std::cout << "ERROR: failed to persist merged index file: " << table_file.location_ + << ", possible out of disk space" << std::endl; + + return Status::Error(msg); + } + + //step 4: update table files state if (index_size >= options_.index_trigger_size) { table_file.file_type_ = meta::TableFileSchema::TO_INDEX; } else { @@ -471,7 +489,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 1; + int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; } diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index f0bcf82ce8..0fe1f76d29 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -131,7 +131,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab } } - return Status::Error("Table file doesn't exist: " + table_file.file_id_); + std::string msg = "Table file doesn't exist: " + table_file.file_id_; + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 2c319396d8..4b242f230d 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -46,7 +46,20 @@ std::shared_ptr IndexLoadTask::Execute() { ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType)file_->engine_type_); - index_ptr->Load(); + + try { + index_ptr->Load(); + } catch (std::exception& ex) { + //typical error: out of disk space or permition denied + std::string msg = "Failed to load index file: " + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + + for(auto& context : search_contexts_) { + context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed + } + + return nullptr; + } size_t file_size = index_ptr->PhysicalSize();