admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

MS-574 add GET config APIs

Browse Source 
Former-commit-id: 6011f32d650d26367ab84f4bf8521889881f2629
This commit is contained in:
yudong.cai 2019-09-21 15:02:35 +08:00
parent 82e37e11ba
commit 22563d83d7
13 changed files with 280 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cpp/conf/server_config.template
View File

@ -5,13 +5,13 @@ server_config:
time_zone: UTC+8
db_config:
db_path: @MILVUS_DB_PATH@ # milvus database path
db_slave_path: # secondary database path, split by semicolon
path: @MILVUS_DB_PATH@ # milvus database path
slave_path: # secondary database path, split by semicolon
# URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite
db_backend_url: sqlite://:@:/
backend_url: sqlite://:@:/
archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit
archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit
@ -27,7 +27,7 @@ metric_config:
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory size used for cache
cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered
insert_immediately: false # whether load data into cache when insert
cache_insert_data: false # whether load data into cache when insert
engine_config:
blas_threshold: 20

17
cpp/src/cache/CpuCacheMgr.cpp vendored
View File

@ -29,19 +29,16 @@ namespace {
}
CpuCacheMgr::CpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
int64_t cap =
config.GetInt64Value(server::CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT));
cap *= unit;
server::ServerConfig& config = server::ServerConfig::GetInstance();
int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
double free_percent =
config.GetDoubleValue(server::CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT));
if(free_percent > 0.0 && free_percent <= 1.0) {
cache_->set_freemem_percent(free_percent);
float cpu_mem_threshold = config.GetCacheConfigCpuMemThreshold();
if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(cpu_mem_threshold);
} else {
SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent <<
", defaultly set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold
<< ", by default set to " << cache_->freemem_percent();
}
}

17
cpp/src/cache/GpuCacheMgr.cpp vendored
View File

@ -33,20 +33,17 @@ namespace {
}
GpuCacheMgr::GpuCacheMgr() {
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
server::ServerConfig& config = server::ServerConfig::GetInstance();
int64_t cap =
config.GetInt64Value(server::CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT));
cap *= G_BYTE;
int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
double free_percent =
config.GetDoubleValue(server::CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT));
if (free_percent > 0.0 && free_percent <= 1.0) {
cache_->set_freemem_percent(free_percent);
float gpu_mem_threshold = config.GetCacheConfigGpuMemThreshold();
if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(gpu_mem_threshold);
} else {
SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent <<
", defaultly set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold
<< ", by default set to " << cache_->freemem_percent();
}
}

3
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -335,8 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_DB);
gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT));
gpu_num_ = config.GetDBConfigBuildIndexGPU();
return Status::OK();
}

6
cpp/src/metrics/Metrics.cpp
View File

@ -16,6 +16,7 @@
// under the License.
#include "Metrics.h"
#include "server/ServerConfig.h"
#include "PrometheusMetrics.h"
@ -31,9 +32,8 @@ Metrics::GetInstance() {
MetricsBase &
Metrics::CreateMetricsCollector() {
ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC);
std::string collector_type_str =
config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT);
ServerConfig &config = ServerConfig::GetInstance();
std::string collector_type_str = config.GetMetricConfigCollector();
if (collector_type_str == "prometheus") {
return PrometheusMetrics::GetInstance();

9
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -29,15 +29,12 @@ namespace server {
ErrorCode
PrometheusMetrics::Init() {
try {
ConfigNode &metric_config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC);
startup_ =
metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT));
ServerConfig &config = ServerConfig::GetInstance();
startup_ = config.GetMetricConfigAutoBootup();
if (!startup_) return SERVER_SUCCESS;
// Following should be read from config file.
ConfigNode &prometheus_config = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS);
const std::string bind_address =
prometheus_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT);
const std::string bind_address = config.GetMetricConfigPrometheusPort();
const std::string uri = std::string("/tmp/metrics");
const std::size_t num_threads = 2;

7
cpp/src/scheduler/SchedInst.cpp
View File

@ -38,10 +38,9 @@ std::mutex JobMgrInst::mutex_;
void
load_simple_config() {
server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE);
auto mode = config.GetValue(server::CONFIG_RESOURCE_MODE, server::CONFIG_RESOURCE_MODE_DEFAULT);
auto pool = config.GetSequence(server::CONFIG_RESOURCE_POOL);
server::ServerConfig &config = server::ServerConfig::GetInstance();
auto mode = config.GetResourceConfigMode();
auto pool = config.GetResourceConfigPool();
bool cpu = false;
std::set<uint64_t> gpu_ids;
for (auto &resource : pool) {

34
cpp/src/server/DBWrapper.cpp
View File

@ -35,23 +35,20 @@ DBWrapper::DBWrapper() {
}
Status DBWrapper::StartService() {
ServerConfig& config = ServerConfig::GetInstance();
//db config
engine::DBOptions opt;
ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_BACKEND_URL);
std::string db_path = db_config.GetValue(CONFIG_DB_PATH);
opt.meta.path = db_path + "/db";
opt.meta.backend_uri = config.GetDBConfigBackendUrl();
opt.meta.path = config.GetDBConfigPath() + "/db";
std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH);
std::string db_slave_path = config.GetDBConfigSlavePath();
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths);
// cache config
ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE);
opt.insert_cache_immediately_ =
cache_config.GetBoolValue(CONFIG_CACHE_INSERT_IMMEDIATELY, std::stoi(CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT));
opt.insert_cache_immediately_ = config.GetCacheConfigCacheInsertData();
ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
std::string mode = serverConfig.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT);
std::string mode = config.GetServerConfigMode();
if (mode == "single") {
opt.mode = engine::DBOptions::MODE::SINGLE;
}
@ -67,9 +64,7 @@ Status DBWrapper::StartService() {
}
// engine config
ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE);
int32_t omp_thread =
engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT));
int32_t omp_thread = config.GetEngineConfigOmpThreadNum();
if(omp_thread > 0) {
omp_set_num_threads(omp_thread);
SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread;
@ -82,19 +77,16 @@ Status DBWrapper::StartService() {
}
//init faiss global variable
faiss::distance_compute_blas_threshold =
engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT));
faiss::distance_compute_blas_threshold = config.GetEngineConfigBlasThreshold();
//set archive config
engine::ArchiveConf::CriteriaT criterial;
int64_t disk =
db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT));
int64_t days =
db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT));
if(disk > 0) {
int32_t disk = config.GetDBConfigArchiveDiskThreshold();
int32_t days = config.GetDBConfigArchiveDaysThreshold();
if (disk > 0) {
criterial[engine::ARCHIVE_CONF_DISK] = disk;
}
if(days > 0) {
if (days > 0) {
criterial[engine::ARCHIVE_CONF_DAYS] = days;
}
opt.meta.archive_conf.SetCriterias(criterial);

9
cpp/src/server/Server.cpp
View File

@ -162,9 +162,7 @@ Server::Start() {
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT);
std::string time_zone = config.GetServerConfigTimeZone();
if (time_zone.length() == 3) {
time_zone = "CUT";
} else {
@ -232,8 +230,9 @@ Server::Stop() {
ErrorCode
Server::LoadConfig() {
ServerConfig::GetInstance().LoadConfigFile(config_filename_);
auto status = ServerConfig::GetInstance().ValidateConfig();
ServerConfig& server_config = ServerConfig::GetInstance();
server_config.LoadConfigFile(config_filename_);
auto status = server_config.ValidateConfig();
if (!status.ok()) {
std::cerr << "Failed to load config file: " << config_filename_ << std::endl;
exit(0);

220
cpp/src/server/ServerConfig.cpp
View File

@ -42,30 +42,29 @@ ServerConfig::GetInstance() {
}
Status
ServerConfig::LoadConfigFile(const std::string &config_filename) {
std::string filename = config_filename;
ServerConfig::LoadConfigFile(const std::string &filename) {
if (filename.empty()) {
std::cerr << "ERROR: a config file is required" << std::endl;
exit(1);//directly exit program if config file not specified
std::cerr << "ERROR: need specify config file" << std::endl;
exit(1);
}
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
struct stat dirStat;
int statOK = stat(filename.c_str(), &dirStat);
if (statOK != 0) {
std::cerr << "ERROR: " << filename << " not found!" << std::endl;
exit(1);//directly exit program if config file not found
std::cerr << "ERROR: Config file not exist: " << filename << std::endl;
exit(1);
}
try {
ConfigMgr *mgr = const_cast<ConfigMgr *>(ConfigMgr::GetInstance());
ErrorCode err = mgr->LoadConfigFile(filename);
if (err != 0) {
std::cerr << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal
std::cerr << "Server failed to load config file: " << filename << std::endl;
exit(1);
}
}
catch (YAML::Exception &e) {
std::cerr << "Server failed to load config file: " << std::endl;
exit(1);//directly exit program if the config file is illegal
std::cerr << "Server failed to load config file: " << filename << std::endl;
exit(1);
}
return Status::OK();
@ -73,27 +72,25 @@ ServerConfig::LoadConfigFile(const std::string &config_filename) {
Status
ServerConfig::ValidateConfig() {
bool okay = true;
if (!CheckServerConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail");
}
if (!CheckDBConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail");
}
if (!CheckMetricConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail");
}
if (!CheckCacheConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail");
}
if (!CheckEngineConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail");
}
if (!CheckResourceConfig().ok()) {
okay = false;
return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail");
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Config validation not pass"));
return Status::OK();
}
Status
@ -201,18 +198,18 @@ ServerConfig::CheckDBConfig() {
okay = false;
}
std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT);
if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) {
std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl;
std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT);
if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) {
std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl;
okay = false;
}
else {
uint64_t insert_buffer_size = (uint64_t) std::stol(insert_buffer_size_str);
insert_buffer_size *= GB;
uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str);
buffer_size *= GB;
unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if (insert_buffer_size >= total_mem) {
std::cerr << "ERROR: insert_buffer_size exceed system memory" << std::endl;
if (buffer_size >= total_mem) {
std::cerr << "ERROR: buffer_size exceed system memory" << std::endl;
okay = false;
}
}
@ -317,7 +314,7 @@ ServerConfig::CheckCacheConfig() {
}
std::string insert_cache_immediately_str =
cache_config.GetValue(CONFIG_CACHE_INSERT_IMMEDIATELY, CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT);
cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT);
if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) {
std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl;
okay = false;
@ -629,6 +626,171 @@ ServerConfig::GetConfig(const std::string &name) {
return root_node.GetChild(name);
}
/* server config */
std::string
ServerConfig::GetServerConfigAddress() {
ConfigNode server_config = GetConfig(CONFIG_SERVER);
return server_config.GetValue(CONFIG_SERVER_ADDRESS,
CONFIG_SERVER_ADDRESS_DEFAULT);
}
std::string
ServerConfig::GetServerConfigPort() {
ConfigNode server_config = GetConfig(CONFIG_SERVER);
return server_config.GetValue(CONFIG_SERVER_PORT,
CONFIG_SERVER_PORT_DEFAULT);
}
std::string
ServerConfig::GetServerConfigMode() {
ConfigNode server_config = GetConfig(CONFIG_SERVER);
return server_config.GetValue(CONFIG_SERVER_MODE,
CONFIG_SERVER_MODE_DEFAULT);
}
std::string
ServerConfig::GetServerConfigTimeZone() {
ConfigNode server_config = GetConfig(CONFIG_SERVER);
return server_config.GetValue(CONFIG_SERVER_TIME_ZONE,
CONFIG_SERVER_TIME_ZONE_DEFAULT);
}
/* db config */
std::string
ServerConfig::GetDBConfigPath() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetValue(CONFIG_DB_PATH,
CONFIG_DB_PATH_DEFAULT);
}
std::string
ServerConfig::GetDBConfigSlavePath() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetValue(CONFIG_DB_SLAVE_PATH,
CONFIG_DB_SLAVE_PATH_DEFAULT);
}
std::string
ServerConfig::GetDBConfigBackendUrl() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetValue(CONFIG_DB_BACKEND_URL,
CONFIG_DB_BACKEND_URL_DEFAULT);
}
int32_t
ServerConfig::GetDBConfigArchiveDiskThreshold() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD,
std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT));
}
int32_t
ServerConfig::GetDBConfigArchiveDaysThreshold() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD,
std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT));
}
int32_t
ServerConfig::GetDBConfigBufferSize() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE,
std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT));
}
int32_t
ServerConfig::GetDBConfigBuildIndexGPU() {
ConfigNode db_config = GetConfig(CONFIG_DB);
return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU,
std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT));
}
/* metric config */
bool
ServerConfig::GetMetricConfigAutoBootup() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC);
return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP,
std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT));
}
std::string
ServerConfig::GetMetricConfigCollector() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC);
return metric_config.GetValue(CONFIG_METRIC_COLLECTOR,
CONFIG_METRIC_COLLECTOR_DEFAULT);
}
std::string
ServerConfig::GetMetricConfigPrometheusPort() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC);
return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT,
CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT);
}
/* cache config */
int32_t
ServerConfig::GetCacheConfigCpuMemCapacity() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY,
std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT));
}
float
ServerConfig::GetCacheConfigCpuMemThreshold() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD,
std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT));
}
int32_t
ServerConfig::GetCacheConfigGpuMemCapacity() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY,
std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT));
}
float
ServerConfig::GetCacheConfigGpuMemThreshold() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD,
std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT));
}
bool
ServerConfig::GetCacheConfigCacheInsertData() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA,
std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT));
}
/* engine config */
int32_t
ServerConfig::GetEngineConfigBlasThreshold() {
ConfigNode engine_config = GetConfig(CONFIG_ENGINE);
return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD,
std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT));
}
int32_t
ServerConfig::GetEngineConfigOmpThreadNum() {
ConfigNode engine_config = GetConfig(CONFIG_ENGINE);
return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM,
std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT));
}
/* resource config */
std::string
ServerConfig::GetResourceConfigMode() {
ConfigNode resource_config = GetConfig(CONFIG_RESOURCE);
return resource_config.GetValue(CONFIG_RESOURCE_MODE,
CONFIG_RESOURCE_MODE_DEFAULT);
}
std::vector<std::string>
ServerConfig::GetResourceConfigPool() {
ConfigNode resource_config = GetConfig(CONFIG_RESOURCE);
return resource_config.GetSequence(CONFIG_RESOURCE_POOL);
}
}
}

45
cpp/src/server/ServerConfig.h
View File

@ -39,11 +39,11 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
/* db config */
static const char* CONFIG_DB = "db_config";
static const char* CONFIG_DB_PATH = "db_path";
static const char* CONFIG_DB_PATH = "path";
static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus";
static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path";
static const char* CONFIG_DB_SLAVE_PATH = "slave_path";
static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = "";
static const char* CONFIG_DB_BACKEND_URL = "db_backend_url";
static const char* CONFIG_DB_BACKEND_URL = "backend_url";
static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
@ -64,8 +64,8 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold";
static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_INSERT_IMMEDIATELY = "insert_immediately";
static const char* CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT = "0";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0";
/* metric config */
static const char* CONFIG_METRIC = "metric_config";
@ -95,20 +95,51 @@ class ServerConfig {
public:
static ServerConfig &GetInstance();
Status LoadConfigFile(const std::string& config_filename);
Status LoadConfigFile(const std::string& filename);
Status ValidateConfig();
void PrintAll() const;
private:
ConfigNode GetConfig(const std::string& name) const;
ConfigNode& GetConfig(const std::string& name);
private:
Status CheckServerConfig();
Status CheckDBConfig();
Status CheckMetricConfig();
Status CheckCacheConfig();
Status CheckEngineConfig();
Status CheckResourceConfig();
public:
std::string GetServerConfigAddress();
std::string GetServerConfigPort();
std::string GetServerConfigMode();
std::string GetServerConfigTimeZone();
std::string GetDBConfigPath();
std::string GetDBConfigSlavePath();
std::string GetDBConfigBackendUrl();
int32_t GetDBConfigArchiveDiskThreshold();
int32_t GetDBConfigArchiveDaysThreshold();
int32_t GetDBConfigBufferSize();
int32_t GetDBConfigBuildIndexGPU();
bool GetMetricConfigAutoBootup();
std::string GetMetricConfigCollector();
std::string GetMetricConfigPrometheusPort();
int32_t GetCacheConfigCpuMemCapacity();
float GetCacheConfigCpuMemThreshold();
int32_t GetCacheConfigGpuMemCapacity();
float GetCacheConfigGpuMemThreshold();
bool GetCacheConfigCacheInsertData();
int32_t GetEngineConfigBlasThreshold();
int32_t GetEngineConfigOmpThreadNum();
std::string GetResourceConfigMode();
std::vector<std::string>
GetResourceConfigPool();
};
}

8
cpp/src/server/grpc_impl/GrpcServer.cpp
View File

@ -74,12 +74,10 @@ GrpcServer::Stop() {
Status
GrpcServer::StartService() {
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, std::stoi(CONFIG_SERVER_PORT_DEFAULT));
std::string address = config.GetServerConfigAddress();
std::string port = config.GetServerConfigPort();
std::string server_address(address + ":" + std::to_string(port));
std::string server_address(address + ":" + port);
::grpc::ServerBuilder builder;
builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption));

9
cpp/src/wrapper/KnowhereResource.cpp
View File

@ -38,16 +38,13 @@ ErrorCode KnowhereResource::Initialize() {
GpuResourcesArray gpu_resources;
//get build index gpu resource
server::ServerConfig& root_config = server::ServerConfig::GetInstance();
server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB);
server::ServerConfig& config = server::ServerConfig::GetInstance();
int32_t build_index_gpu =
db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, std::stoi(server::CONFIG_DB_BUILD_INDEX_GPU_DEFAULT));
int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU();
gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting()));
//get search gpu resource
server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE);
auto pool = res_config.GetSequence(server::CONFIG_RESOURCE_POOL);
auto pool = config.GetResourceConfigPool();
std::set<uint64_t> gpu_ids;
for (auto &resource : pool) {
if (resource.length() < 4 || resource.substr(0, 3) != "gpu") {