admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-06 19:02:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.1' into 'branch-0.3.1'

Browse Source 
MS-230 - Change parameter name: Maximum_memory to insert_buffer_size

See merge request megasearch/milvus!212

Former-commit-id: df366449f62ae29b713edf0b4961ed79c3726ab8
This commit is contained in:
peng.xu 2019-07-15 14:06:11 +08:00
commit c857197501
7 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cpp/CHANGELOG.md
View File

@ -9,13 +9,14 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-148 - Disable cleanup if mode is read only
- MS-149 - Fixed searching only one index file issue in distributed mode
- MS-153 - fix c_str error when connecting to MySQL
- MS-157 - fix changelog
- MS-190 - use env variable to switch mem manager and fix cmake
- MS-153 - Fix c_str error when connecting to MySQL
- MS-157 - Fix changelog
- MS-190 - Use env variable to switch mem manager and fix cmake
- MS-217 - Fix SQ8 row count bug
- MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists
- MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0
- MS-233 - remove mem manager log
- MS-232 - Add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0
- MS-233 - Remove mem manager log
- MS-230 - Change parameter name: Maximum_memory to insert_buffer_size
## Improvement
- MS-156 - Add unittest for merge result functions

4
cpp/conf/server_config.template
View File

@ -16,8 +16,8 @@ db_config:
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of maximum_memory and cpu_cache_capacity should be less than total memory
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory
metric_config:
is_startup: off # if monitoring start: on, off

2
cpp/src/CMakeLists.txt
View File

@ -189,4 +189,4 @@ install(FILES
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
DESTINATION lib) #need to copy libmysqlpp.so
#add_subdirectory(sdk)
add_subdirectory(sdk)

2
cpp/src/db/NewMemManager.cpp
View File

@ -25,7 +25,7 @@ Status NewMemManager::InsertVectors(const std::string &table_id_,
const float *vectors_,
IDNumbers &vector_ids_) {
while (GetCurrentMem() > options_.maximum_memory) {
while (GetCurrentMem() > options_.insert_buffer_size) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}

2
cpp/src/db/Options.h
View File

@ -63,7 +63,7 @@ struct Options {
size_t index_trigger_size = ONE_GB; //unit: byte
DBMetaOptions meta;
int mode = MODE::SINGLE;
size_t maximum_memory = 4 * ONE_GB;
size_t insert_buffer_size = 4 * ONE_GB;
}; // Options

10
cpp/src/server/DBWrapper.cpp
View File

@ -28,12 +28,12 @@ DBWrapper::DBWrapper() {
if(index_size > 0) {//ensure larger than zero, unit is MB
opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
}
int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY);
if (maximum_memory > 1.0) {
opt.maximum_memory = maximum_memory * engine::ONE_GB;
int64_t insert_buffer_size = config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
if (insert_buffer_size >= 1) {
opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB;
}
else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default
std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl;
else {
std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl;
kill(0, SIGUSR1);
}

2
cpp/src/server/ServerConfig.h
View File

@ -27,7 +27,7 @@ static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path";
static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const std::string CONFIG_MAXMIMUM_MEMORY = "maximum_memory";
static const std::string CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const std::string CONFIG_LOG = "log_config";