diff --git a/CHANGELOG.md b/CHANGELOG.md index 2956b867fb..20b472b2c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#255 - Add ivfsq8 test report detailed version - \#260 - C++ SDK README - \#266 - Rpc request source code refactor +- \#274 - Logger the time cost during preloading data - \#275 - Rename C++ SDK IndexType - \#284 - Change C++ SDK to shared library - \#306 - Use int64 for all config integer diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 0b3309d84e..8e08a850f8 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -182,7 +182,7 @@ DBImpl::PreloadTable(const std::string& table_id) { return SHUTDOWN_ERROR; } - // get all table files from parent table + // step 1: get all table files from parent table meta::DatesT dates; std::vector ids; meta::TableFilesSchema files_array; @@ -191,7 +191,7 @@ DBImpl::PreloadTable(const std::string& table_id) { return status; } - // get files from partition tables + // step 2: get files from partition tables std::vector partiton_array; status = meta_ptr_->ShowPartitions(table_id, partiton_array); for (auto& schema : partiton_array) { @@ -203,6 +203,10 @@ DBImpl::PreloadTable(const std::string& table_id) { int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; + // step 3: load file one by one + ENGINE_LOG_DEBUG << "Begin pre-load table:" + table_id + ", totally " << files_array.size() + << " files need to be pre-loaded"; + TimeRecorderAuto rc("Pre-load table:" + table_id); for (auto& file : files_array) { ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); @@ -213,10 +217,12 @@ DBImpl::PreloadTable(const std::string& table_id) { size += engine->PhysicalSize(); if (size > available_size) { + ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full"; return Status(SERVER_CACHE_FULL, "Cache is full"); } else { try { - // step 1: load index + std::string msg = "Pre-loaded file: " + file.file_id_ + " size: " + std::to_string(file.file_size_); + TimeRecorderAuto rc_1(msg); engine->Load(true); } catch (std::exception& ex) { std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); diff --git a/core/src/utils/TimeRecorder.cpp b/core/src/utils/TimeRecorder.cpp index f3061d9d2b..2072346942 100644 --- a/core/src/utils/TimeRecorder.cpp +++ b/core/src/utils/TimeRecorder.cpp @@ -96,4 +96,11 @@ TimeRecorder::ElapseFromBegin(const std::string& msg) { return span; } +TimeRecorderAuto::TimeRecorderAuto(const std::string& header, int64_t log_level) : TimeRecorder(header, log_level) { +} + +TimeRecorderAuto::~TimeRecorderAuto() { + ElapseFromBegin("totally cost"); +} + } // namespace milvus diff --git a/core/src/utils/TimeRecorder.h b/core/src/utils/TimeRecorder.h index cc0a86fbe0..8f6990f482 100644 --- a/core/src/utils/TimeRecorder.h +++ b/core/src/utils/TimeRecorder.h @@ -28,7 +28,7 @@ class TimeRecorder { public: explicit TimeRecorder(const std::string& header, int64_t log_level = 1); - ~TimeRecorder(); // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 + virtual ~TimeRecorder(); // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 double RecordSection(const std::string& msg); @@ -50,4 +50,11 @@ class TimeRecorder { int64_t log_level_; }; +class TimeRecorderAuto : public TimeRecorder { + public: + explicit TimeRecorderAuto(const std::string& header, int64_t log_level = 1); + + ~TimeRecorderAuto(); +}; + } // namespace milvus