diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index c918c74908..b59fa78b67 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -11,6 +11,8 @@ db_config: #Currently supports mysql or sqlite db_backend_url: mysql://root:1234@:/test # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 6549a1740a..9a44820c7a 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -656,7 +656,7 @@ Status DBMetaImpl::Archive() { for (auto kv : criterias) { auto &criteria = kv.first; auto &limit = kv.second; - if (criteria == "days") { + if (criteria == engine::ARCHIVE_CONF_DAYS) { long usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); try { @@ -672,11 +672,11 @@ Status DBMetaImpl::Archive() { return HandleException("Encounter exception when update table files", e); } } - if (criteria == "disk") { + if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); } } diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ff6404751f..5f591dcb0f 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) ParseCritirias(criterias); } +void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for(auto& pair : criterial) { + criterias_[pair.first] = pair.second; + } +} + void ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 039134ebaa..dbe80f8d5f 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024; static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static const std::string ARCHIVE_CONF_DISK = "disk"; +static const std::string ARCHIVE_CONF_DAYS = "days"; +static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; + struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = "disk:512"); + ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT); const std::string& GetType() const { return type_; } const CriteriaT GetCriterias() const { return criterias_; } + void SetCriterias(const ArchiveConf::CriteriaT& criterial); + private: void ParseCritirias(const std::string& type); void ParseType(const std::string& criterias); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 6ad308a8d6..c36bf2621e 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -27,6 +27,19 @@ DBWrapper::DBWrapper() { opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); // std::cout << "mode = " << opt.mode << std::endl; + //set archive config + engine::ArchiveConf::CriteriaT criterial; + int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); + int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + if(disk > 0) { + criterial[engine::ARCHIVE_CONF_DISK] = disk; + } + if(days > 0) { + criterial[engine::ARCHIVE_CONF_DAYS] = days; + } + opt.meta.archive_conf.SetCriterias(criterial); + + //create db root folder CommonUtil::CreateDirectory(opt.meta.path); zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index aa77b081d7..f337275a46 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -25,6 +25,8 @@ static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const std::string CONFIG_LOG = "log_config";