diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 4ace334778..1c26c0aa52 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -8,6 +8,8 @@ db_config: db_path: /tmp/milvus # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 779307e8a6..1a8419eac2 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -655,7 +655,7 @@ Status DBMetaImpl::Archive() { for (auto kv : criterias) { auto &criteria = kv.first; auto &limit = kv.second; - if (criteria == "days") { + if (criteria == engine::ARCHIVE_CONF_DAYS) { long usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); try { @@ -671,11 +671,11 @@ Status DBMetaImpl::Archive() { return HandleException("Encounter exception when update table files", e); } } - if (criteria == "disk") { + if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); } } diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ff6404751f..5f591dcb0f 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) ParseCritirias(criterias); } +void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for(auto& pair : criterial) { + criterias_[pair.first] = pair.second; + } +} + void ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 339044d9dd..919d21709c 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024; static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static const std::string ARCHIVE_CONF_DISK = "disk"; +static const std::string ARCHIVE_CONF_DAYS = "days"; +static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; + struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = "disk:512"); + ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT); const std::string& GetType() const { return type_; } const CriteriaT GetCriterias() const { return criterias_; } + void SetCriterias(const ArchiveConf::CriteriaT& criterial); + private: void ParseCritirias(const std::string& type); void ParseType(const std::string& criterias); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 4cae31ea6b..2a583a660d 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -24,6 +24,19 @@ DBWrapper::DBWrapper() { opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + //set archive config + engine::ArchiveConf::CriteriaT criterial; + int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); + int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + if(disk > 0) { + criterial[engine::ARCHIVE_CONF_DISK] = disk; + } + if(days > 0) { + criterial[engine::ARCHIVE_CONF_DAYS] = days; + } + opt.meta.archive_conf.SetCriterias(criterial); + + //create db root folder CommonUtil::CreateDirectory(opt.meta.path); zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index dd7c9d2966..abf898bacf 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -24,6 +24,8 @@ static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const std::string CONFIG_LOG = "log_config";