admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

archive config

Browse Source 
Former-commit-id: c54cb683d0eae72f9eaa82c6159dc6bb411d6edf
This commit is contained in:
starlord 2019-06-27 11:39:27 +08:00 committed by jinhai
parent de751bd680
commit e8c4390327
6 changed files with 33 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/conf/server_config.yaml
View File

@ -11,6 +11,8 @@ db_config:
#Currently supports mysql or sqlite
db_backend_url: mysql://root:1234@:/test # meta database uri
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB
archive_days_threshold: 30 # files older than x days will be archived, unit: day
metric_config:
is_startup: off # if monitoring start: on, off

6
cpp/src/db/DBMetaImpl.cpp
View File

@ -656,7 +656,7 @@ Status DBMetaImpl::Archive() {
for (auto kv : criterias) {
auto &criteria = kv.first;
auto &limit = kv.second;
if (criteria == "days") {
if (criteria == engine::ARCHIVE_CONF_DAYS) {
long usecs = limit * D_SEC * US_PS;
long now = utils::GetMicroSecTimeStamp();
try {
@ -672,11 +672,11 @@ Status DBMetaImpl::Archive() {
return HandleException("Encounter exception when update table files", e);
}
}
if (criteria == "disk") {
if (criteria == engine::ARCHIVE_CONF_DISK) {
uint64_t sum = 0;
Size(sum);
auto to_delete = (sum - limit * G);
int64_t to_delete = (int64_t)sum - limit * G;
DiscardFiles(to_delete);
}
}

6
cpp/src/db/Options.cpp
View File

@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias)
ParseCritirias(criterias);
}
void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
for(auto& pair : criterial) {
criterias_[pair.first] = pair.second;
}
}
void ArchiveConf::ParseCritirias(const std::string& criterias) {
std::stringstream ss(criterias);
std::vector<std::string> tokens;

8
cpp/src/db/Options.h
View File

@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024;
static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
static const std::string ARCHIVE_CONF_DISK = "disk";
static const std::string ARCHIVE_CONF_DAYS = "days";
static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;
ArchiveConf(const std::string& type, const std::string& criterias = "disk:512");
ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT);
const std::string& GetType() const { return type_; }
const CriteriaT GetCriterias() const { return criterias_; }
void SetCriterias(const ArchiveConf::CriteriaT& criterial);
private:
void ParseCritirias(const std::string& type);
void ParseType(const std::string& criterias);

13
cpp/src/server/DBWrapper.cpp
View File

@ -27,6 +27,19 @@ DBWrapper::DBWrapper() {
opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single");
// std::cout << "mode = " << opt.mode << std::endl;
//set archive config
engine::ArchiveConf::CriteriaT criterial;
int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0);
int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0);
if(disk > 0) {
criterial[engine::ARCHIVE_CONF_DISK] = disk;
}
if(days > 0) {
criterial[engine::ARCHIVE_CONF_DAYS] = days;
}
opt.meta.archive_conf.SetCriterias(criterial);
//create db root folder
CommonUtil::CreateDirectory(opt.meta.path);
zilliz::milvus::engine::DB::Open(opt, &db_);

2
cpp/src/server/ServerConfig.h
View File

@ -25,6 +25,8 @@ static const std::string CONFIG_DB = "db_config";
static const std::string CONFIG_DB_URL = "db_backend_url";
static const std::string CONFIG_DB_PATH = "db_path";
static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const std::string CONFIG_LOG = "log_config";