From ba67c75933fb6a0012b174e14e16b8b32305f0a4 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 16 Jun 2019 18:34:51 +0800 Subject: [PATCH] Enhancement for MemVector size control Former-commit-id: b6648148d8d9ba3f894c6c8aace5c1351f4f5126 --- cpp/conf/server_config.yaml | 1 + cpp/conf/server_config_template.yaml | 1 + cpp/src/db/DBMetaImpl.cpp | 3 ++- cpp/src/db/Options.h | 4 ++-- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 7 ++++--- cpp/src/server/RequestTask.cpp | 4 ++++ cpp/src/server/ServerConfig.h | 1 + 7 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 6d7c9f6d0d..ae3907fbb9 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -10,6 +10,7 @@ db_config: db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 + index_trigger_size: 1024 #unit: MB metric_config: is_startup: true # true is on, false is off diff --git a/cpp/conf/server_config_template.yaml b/cpp/conf/server_config_template.yaml index 59096cd8b6..c0c9672eb3 100644 --- a/cpp/conf/server_config_template.yaml +++ b/cpp/conf/server_config_template.yaml @@ -10,6 +10,7 @@ db_config: db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 + index_trigger_size: 1024 #unit: MB metric_config: is_startup: true # true is on, false is off diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 864926b6c1..ceaf0f8c68 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -515,7 +515,8 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, &TableFileSchema::size_, &TableFileSchema::date_), where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and - c(&TableFileSchema::table_id_) == table_id)); + c(&TableFileSchema::table_id_) == table_id), + order_by(&TableFileSchema::size_).desc()); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 31f5cb1d87..2c76259214 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -40,9 +40,9 @@ struct DBMetaOptions { struct Options { Options(); - uint16_t memory_sync_interval = 1; + uint16_t memory_sync_interval = 1; //unit: second uint16_t merge_trigger_number = 2; - size_t index_trigger_size = 1024*1024*1024; + size_t index_trigger_size = 1024*1024*1024; //unit: byte Env* env; DBMetaOptions meta; }; // Options diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index b25785f4ad..825923b1a4 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -20,6 +20,7 @@ namespace { static constexpr int64_t TOTAL_ROW_COUNT = 100000; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different + static constexpr int64_t ADD_VECTOR_LOOP = 1; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -156,9 +157,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//create table TableSchema tb_schema = BuildTableSchema(); - PrintTableSchema(tb_schema); Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.ToString() << std::endl; + PrintTableSchema(tb_schema); } {//describe table @@ -168,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { PrintTableSchema(tb_schema); } - {//add vectors + for(int i = 0; i < ADD_VECTOR_LOOP; i++){//add vectors std::vector record_array; - BuildVectors(0, TOTAL_ROW_COUNT, record_array); + BuildVectors(i*TOTAL_ROW_COUNT, (i+1)*TOTAL_ROW_COUNT, record_array); std::vector record_ids; Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); std::cout << "AddVector function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 5715b55cb0..2f0d50437f 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -36,6 +36,10 @@ namespace { std::string db_path = config.GetValue(CONFIG_DB_PATH); opt.memory_sync_interval = (uint16_t)config.GetInt32Value(CONFIG_DB_FLUSH_INTERVAL, 10); opt.meta.path = db_path + "/db"; + int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + if(index_size > 0) {//ensure larger than zero + opt.index_trigger_size = (size_t)index_size; + } CommonUtil::CreateDirectory(opt.meta.path); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index c7e02d2177..1fed87e2bf 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -25,6 +25,7 @@ static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_FLUSH_INTERVAL = "db_flush_interval"; static const std::string CONFIG_DB_IDMAPPER_MAX_FILE = "idmapper_max_open_file"; +static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_trigger_size"; static const std::string CONFIG_LOG = "log_config";