From a01cdbf886509aff586add3698835ff89778cc98 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 13:47:24 +0800 Subject: [PATCH 1/2] MS-230 - Change parameter name: Maximum_memory to insert_buffer_size Former-commit-id: 8a27b0b4b272a89a2a1e4e4fa78c26baccaf139d --- cpp/conf/server_config.template | 4 ++-- cpp/src/db/NewMemManager.cpp | 2 +- cpp/src/db/Options.h | 2 +- cpp/src/server/DBWrapper.cpp | 10 +++++----- cpp/src/server/ServerConfig.h | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 43b6f3ba16..2dd8daa0f1 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -16,8 +16,8 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of maximum_memory and cpu_cache_capacity should be less than total memory + insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. + # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index 405d3771fd..c7291354ad 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -25,7 +25,7 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, const float *vectors_, IDNumbers &vector_ids_) { - while (GetCurrentMem() > options_.maximum_memory) { + while (GetCurrentMem() > options_.insert_buffer_size) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index a12afbfd80..efc5ee3604 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -63,7 +63,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; - size_t maximum_memory = 4 * ONE_GB; + size_t insert_buffer_size = 4 * ONE_GB; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c01e1acda3..6c67176571 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,12 +28,12 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY); - if (maximum_memory > 1.0) { - opt.maximum_memory = maximum_memory * engine::ONE_GB; + int64_t insert_buffer_size = config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); + if (insert_buffer_size >= 1) { + opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB; } - else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default - std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; + else { + std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index d3fb77a8cb..bc202adcf6 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,7 +27,7 @@ static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const std::string CONFIG_MAXMIMUM_MEMORY = "maximum_memory"; +static const std::string CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const std::string CONFIG_LOG = "log_config"; From de09ef53a9a7ba976a29db45218d744ae704c354 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 13:47:36 +0800 Subject: [PATCH 2/2] MS-230 - Change parameter name: Maximum_memory to insert_buffer_size Former-commit-id: 1263b0c2edceb235163c48b34af681c3edb48d16 --- cpp/CHANGELOG.md | 11 ++++++----- cpp/src/CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 4a3288526c..3ac9db25a1 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,13 +9,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog -- MS-190 - use env variable to switch mem manager and fix cmake +- MS-153 - Fix c_str error when connecting to MySQL +- MS-157 - Fix changelog +- MS-190 - Use env variable to switch mem manager and fix cmake - MS-217 - Fix SQ8 row count bug - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists -- MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 -- MS-233 - remove mem manager log +- MS-232 - Add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 +- MS-233 - Remove mem manager log +- MS-230 - Change parameter name: Maximum_memory to insert_buffer_size ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d0029d5175..00e2e1e87d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION lib) #need to copy libmysqlpp.so -#add_subdirectory(sdk) +add_subdirectory(sdk)