diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index a3acda41d2..5b68e54249 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -176,14 +176,14 @@ Cache::print() { { std::lock_guard lock(mutex_); cache_count = lru_.size(); +#if 0 + for (auto it = lru_.begin(); it != lru_.end(); ++it) { + SERVER_LOG_DEBUG << it->first; + } +#endif } SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; -#if 1 - for (auto it = lru_.begin(); it != lru_.end(); ++it) { - SERVER_LOG_DEBUG << it->first; - } -#endif SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 11f45501ce..c5e04fb5c6 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -722,7 +722,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - ENGINE_LOG_ERROR << "ERROR: failed to persist merged file: " << table_file.location_ + ENGINE_LOG_ERROR << "Failed to persist merged file: " << table_file.location_ << ", possible out of disk space or memory"; return status; @@ -803,6 +803,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::H_SEC; } + meta_ptr_->CleanUpFilesWithTTL(ttl, &ongoing_files_checker_); } @@ -839,14 +840,13 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - // ENGINE_LOG_TRACE << "Background build index thread start"; - std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status = index_failed_checker_.IgnoreFailedIndexFiles(to_index_files); if (!to_index_files.empty()) { + ENGINE_LOG_DEBUG << "Background build index thread begin"; status = ongoing_files_checker_.MarkOngoingFiles(to_index_files); // step 2: put build index task to scheduler @@ -870,17 +870,15 @@ DBImpl::BackgroundBuildIndex() { index_failed_checker_.MarkFailedIndexFile(file_schema); } else { - index_failed_checker_.MarkSucceedIndexFile(file_schema); ENGINE_LOG_DEBUG << "Building index job " << job->id() << " succeed."; - } - } - status = ongoing_files_checker_.UnmarkOngoingFiles(to_index_files); + index_failed_checker_.MarkSucceedIndexFile(file_schema); + } + status = ongoing_files_checker_.UnmarkOngoingFile(file_schema); + } ENGINE_LOG_DEBUG << "Background build index thread finished"; } - - // ENGINE_LOG_TRACE << "Background build index thread exit"; } Status diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index a87690c6d6..6bb0b97817 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -463,7 +463,7 @@ ExecutionEngineImpl::Merge(const std::string& location) { if (auto file_index = std::dynamic_pointer_cast(to_merge)) { auto status = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); if (!status.ok()) { - ENGINE_LOG_ERROR << "Merge: Add Error"; + ENGINE_LOG_ERROR << "Failed to merge: " << location << " to: " << location_; } return status; } else { diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7f7bd704d5..96e7109c93 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1800,7 +1800,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { mysqlpp::Query query = connectionPtr->query(); query << "SELECT id, table_id, file_id, date" - << " FROM " << META_TABLEFILES << " WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " FROM " << META_TABLEFILES << " WHERE file_type IN (" + << std::to_string(TableFileSchema::TO_DELETE) << "," + << std::to_string(TableFileSchema::BACKUP) << ")" << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << query.str(); @@ -1810,11 +1812,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { TableFileSchema table_file; std::vector idsToDelete; + int64_t clean_files = 0; for (auto& resRow : res) { table_file.id_ = resRow["id"]; // implicit conversion resRow["table_id"].to_string(table_file.table_id_); resRow["file_id"].to_string(table_file.file_id_); table_file.date_ = resRow["date"]; + table_file.file_type_ = resRow["file_type"]; // check if the file can be deleted if (filter && filter->IsIgnored(table_file)) { @@ -1823,15 +1827,21 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { continue; // ignore this file, don't delete it } - // delete file from disk storage - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; - // erase file data from cache + // because GetTableFilePath won't able to generate file path after the file is deleted + utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); - idsToDelete.emplace_back(std::to_string(table_file.id_)); - table_ids.insert(table_file.table_id_); + if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + table_ids.insert(table_file.table_id_); + } + + clean_files++; } // delete file from meta @@ -1852,8 +1862,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { } } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; + if (clean_files > 0) { + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; } } // Scoped Connection } catch (std::exception& e) { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 2a3867574f..846655676d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1302,6 +1302,11 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { try { server::MetricCollector metric; + std::vector file_types = { + (int)TableFileSchema::TO_DELETE, + (int)TableFileSchema::BACKUP, + }; + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); @@ -1309,21 +1314,23 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, + &TableFileSchema::file_type_, &TableFileSchema::date_), where( - c(&TableFileSchema::file_type_) == - (int)TableFileSchema::TO_DELETE + in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::updated_time_) < now - seconds * US_PS)); + int64_t clean_files = 0; auto commited = ConnectorPtr->transaction([&]() mutable { TableFileSchema table_file; for (auto& file : files) { table_file.id_ = std::get<0>(file); table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); - table_file.date_ = std::get<3>(file); + table_file.file_type_ = std::get<3>(file); + table_file.date_ = std::get<4>(file); // check if the file can be deleted if (filter && filter->IsIgnored(table_file)) { @@ -1332,17 +1339,23 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { continue; // ignore this file, don't delete it } - // delete file from meta - ConnectorPtr->remove(table_file.id_); - - // delete file from disk storage - utils::DeleteTableFilePath(options_, table_file); - - // erase from cache + // erase from cache, must do this before file deleted, + // because GetTableFilePath won't able to generate file path after the file is deleted + utils::GetTableFilePath(options_, table_file); server::CommonUtil::EraseFromCache(table_file.location_); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; - table_ids.insert(table_file.table_id_); + if (table_file.file_type_ == (int)TableFileSchema::TO_DELETE) { + // delete file from meta + ConnectorPtr->remove(table_file.id_); + + // delete file from disk storage + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; + table_ids.insert(table_file.table_id_); + } + + clean_files++; } return true; }); @@ -1351,8 +1364,8 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds, CleanUpFilter* filter) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if (files.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; + if (clean_files > 0) { + ENGINE_LOG_DEBUG << "Clean " << clean_files << " files deleted in " << seconds << " seconds"; } } catch (std::exception& e) { return HandleException("Encounter exception when clean table files", e.what()); diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index cfadb2fcc4..cdfae8f1e5 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -229,7 +229,7 @@ CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { void CommonUtil::EraseFromCache(const std::string& item_key) { if (item_key.empty()) { - // SERVER_LOG_ERROR << "Empty key cannot be erased from cache"; + SERVER_LOG_ERROR << "Empty key cannot be erased from cache"; return; }