From 17ec3d5e21cef9e82067a5afe3364ad5cee89e9a Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 10 Mar 2020 18:58:50 +0800 Subject: [PATCH 1/3] #1609 Refine Compact function Signed-off-by: groot --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 66 +++++++++++------------------------------- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 063814d567..b6b4119fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Please mark all change in change log and use the issue from GitHub - \#1590 Server down caused by failure to write file during concurrent mixed operations - \#1598 Server down during mixed operations - \#1601 External link bug in HTTP doc +- \#1609 Refine Compact function ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index f83c0d2b5a..9a59281d83 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -671,26 +671,6 @@ DBImpl::Compact(const std::string& table_id) { ENGINE_LOG_DEBUG << "Compacting table: " << table_id; - /* - // Save table index - TableIndex table_index; - status = DescribeIndex(table_id, table_index); - if (!status.ok()) { - return status; - } - - // Drop all index - status = DropIndex(table_id); - if (!status.ok()) { - return status; - } - - // Then update table index to the previous index - status = UpdateTableIndexRecursively(table_id, table_index); - if (!status.ok()) { - return status; - } - */ // Get files to compact from meta. std::vector file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX, meta::TableFileSchema::FILE_TYPE::BACKUP}; @@ -706,7 +686,6 @@ DBImpl::Compact(const std::string& table_id) { OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact); - meta::TableFilesSchema files_to_update; Status compact_status; for (auto& file : files_to_compact) { // Check if the segment needs compacting @@ -719,52 +698,41 @@ DBImpl::Compact(const std::string& table_id) { if (!status.ok()) { std::string msg = "Failed to load deleted_docs from " + segment_dir; ENGINE_LOG_ERROR << msg; - return Status(DB_ERROR, msg); + OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); + continue; // skip this file and try compact next one } + meta::TableFilesSchema files_to_update; if (deleted_docs->GetSize() != 0) { compact_status = CompactFile(table_id, file, files_to_update); if (!compact_status.ok()) { ENGINE_LOG_ERROR << "Compact failed for segment " << file.segment_id_ << ": " << compact_status.message(); - break; + OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); + continue; // skip this file and try compact next one } } else { + OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact"; + continue; // skip this file and try compact next one + } + + ENGINE_LOG_DEBUG << "Updating meta after compaction..."; + status = meta_ptr_->UpdateTableFiles(files_to_update); + if (!status.ok()) { + compact_status = status; + break; // meta error, could not go on } } + OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); + if (compact_status.ok()) { ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id; } - ENGINE_LOG_DEBUG << "Updating meta after compaction..."; - - /* - // Drop index again, in case some files were in the index building process during compacting - status = DropIndex(table_id); - if (!status.ok()) { - return status; - } - - // Update index - status = UpdateTableIndexRecursively(table_id, table_index); - if (!status.ok()) { - return status; - } - */ - - status = meta_ptr_->UpdateTableFiles(files_to_update); - if (!status.ok()) { - return status; - } - - OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); - - ENGINE_LOG_DEBUG << "Finished updating meta after compaction"; - - return status; + return compact_status; } Status From 88720546031b56f5311fa7ab9aee49cb5f1abef5 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 10 Mar 2020 19:47:23 +0800 Subject: [PATCH 2/3] #1609 Refine Compact function Signed-off-by: groot --- core/src/db/DBImpl.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 9a59281d83..9d10dd5a2a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -687,7 +687,10 @@ DBImpl::Compact(const std::string& table_id) { OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact); Status compact_status; - for (auto& file : files_to_compact) { + for (meta::TableFilesSchema::iterator iter = files_to_compact.begin(); iter != files_to_compact.end();) { + meta::TableFileSchema file = *iter; + iter = files_to_compact.erase(iter); + // Check if the segment needs compacting std::string segment_dir; utils::GetParentPath(file.location_, segment_dir); @@ -724,6 +727,8 @@ DBImpl::Compact(const std::string& table_id) { compact_status = status; break; // meta error, could not go on } + + OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); } OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); From 120ea4d0e388a501773d9ded3a9e97a50f7ebf30 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 10 Mar 2020 19:56:06 +0800 Subject: [PATCH 3/3] #1609 Refine Compact function Signed-off-by: groot --- core/src/db/DBImpl.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 9d10dd5a2a..37a2b5f3e5 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -723,12 +723,11 @@ DBImpl::Compact(const std::string& table_id) { ENGINE_LOG_DEBUG << "Updating meta after compaction..."; status = meta_ptr_->UpdateTableFiles(files_to_update); + OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); if (!status.ok()) { compact_status = status; break; // meta error, could not go on } - - OngoingFileChecker::GetInstance().UnmarkOngoingFile(file); } OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);