From 3c0c4646b07f10a2f77d4bb778f954df8c7487e1 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 7 Nov 2019 15:35:53 +0800 Subject: [PATCH 1/2] issue 229, 230 and 237 --- core/CMakeLists.txt | 24 +- core/build.sh | 16 +- core/cmake/DefineOptions.cmake | 76 ++--- core/conf/server_config.template | 9 +- core/src/CMakeLists.txt | 3 + core/src/index/CMakeLists.txt | 14 +- core/src/index/cmake/DefineOptionsCore.cmake | 19 +- core/src/main.cpp | 10 +- core/src/server/Config.cpp | 143 +++++---- core/src/server/Config.h | 12 + core/src/utils/StringHelpFunctions.cpp | 44 ++- core/src/utils/StringHelpFunctions.h | 5 +- core/unittest/CMakeLists.txt | 1 + core/unittest/server/test_config.cpp | 321 +++++++++++++++++++ core/unittest/server/test_util.cpp | 5 +- 15 files changed, 536 insertions(+), 166 deletions(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 5c63eb39da..cd0aa8196f 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -31,16 +31,16 @@ GET_CURRENT_TIME(BUILD_TIME) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") -MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) +MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) - if(GIT_BRANCH_NAME STREQUAL "") + if (GIT_BRANCH_NAME STREQUAL "") execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) - endif() -ENDMACRO (GET_GIT_BRANCH_NAME) + endif () +ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}") -if(NOT GIT_BRANCH_NAME STREQUAL "") +if (NOT GIT_BRANCH_NAME STREQUAL "") string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME}) endif () @@ -69,7 +69,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") set(MILVUS_VERSION "0.6.0") -endif() +endif () message(STATUS "Build version = ${MILVUS_VERSION}") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY) @@ -118,17 +118,17 @@ include(DefineOptions) include(BuildUtils) include(ThirdPartyPackages) -set(MILVUS_GPU_VERSION false) -if (MILVUS_CPU_VERSION) - message(STATUS "Building Milvus CPU version") - add_compile_definitions("MILVUS_CPU_VERSION") -else () +set(MILVUS_CPU_VERSION false) +if (MILVUS_GPU_VERSION) message(STATUS "Building Milvus GPU version") - set(MILVUS_GPU_VERSION true) add_compile_definitions("MILVUS_GPU_VERSION") enable_language(CUDA) find_package(CUDA 10 REQUIRED) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") +else () + message(STATUS "Building Milvus CPU version") + set(MILVUS_CPU_VERSION true) + add_compile_definitions("MILVUS_CPU_VERSION") endif () if (CMAKE_BUILD_TYPE STREQUAL "Release") diff --git a/core/build.sh b/core/build.sh index 819278b94a..e844528ad3 100755 --- a/core/build.sh +++ b/core/build.sh @@ -12,7 +12,7 @@ USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc -CPU_VERSION="OFF" +GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}" @@ -51,7 +51,7 @@ do c) BUILD_COVERAGE="ON" ;; - g) + z) PROFILING="ON" ;; j) @@ -60,8 +60,8 @@ do x) CUSTOMIZATION="OFF" # force use ori faiss ;; - z) - CPU_VERSION="ON" + g) + GPU_VERSION="ON" ;; m) WITH_MKL="ON" @@ -77,14 +77,14 @@ parameter: -l: run cpplint, clang-format and clang-tidy(default: OFF) -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) --g: profiling(default: OFF) +-z: profiling(default: OFF) -j: use jfrog cache build directory(default: OFF) --z: build pure CPU version(default: OFF) +-g: build GPU version(default: OFF) -m: build with MKL(default: OFF) -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-z] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] " exit 0 ;; @@ -116,7 +116,7 @@ CMAKE_CMD="cmake \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ --DMILVUS_CPU_VERSION=${CPU_VERSION} \ +-DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DBUILD_FAISS_WITH_MKL=${WITH_MKL} \ ../" echo ${CMAKE_CMD} diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index c7f4f73d94..4d5647b24d 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -13,16 +13,16 @@ macro(define_option name description default) endmacro() function(list_join lst glue out) - if("${${lst}}" STREQUAL "") + if ("${${lst}}" STREQUAL "") set(${out} "" PARENT_SCOPE) return() - endif() + endif () list(GET ${lst} 0 joined) list(REMOVE_AT ${lst} 0) - foreach(item ${${lst}}) + foreach (item ${${lst}}) set(joined "${joined}${glue}${item}") - endforeach() + endforeach () set(${out} ${joined} PARENT_SCOPE) endfunction() @@ -35,15 +35,15 @@ macro(define_option_string name description default) set("${name}_OPTION_ENUM" ${ARGN}) list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM") - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set_property(CACHE ${name} PROPERTY STRINGS ${ARGN}) - endif() + endif () endmacro() #---------------------------------------------------------------------- -set_option_category("CPU version") +set_option_category("GPU version") -define_option(MILVUS_CPU_VERSION "Build CPU version only" OFF) +define_option(MILVUS_GPU_VERSION "Build GPU version" OFF) #---------------------------------------------------------------------- set_option_category("Thirdparty") @@ -51,11 +51,11 @@ set_option_category("Thirdparty") set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO") define_option_string(MILVUS_DEPENDENCY_SOURCE - "Method to use for acquiring MILVUS's build dependencies" - "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}" - "AUTO" - "BUNDLED" - "SYSTEM") + "Method to use for acquiring MILVUS's build dependencies" + "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM") define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) @@ -75,14 +75,14 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) if (MILVUS_ENABLE_PROFILING STREQUAL "ON") define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) -endif() +endif () define_option(MILVUS_WITH_GRPC "Build with GRPC" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) #---------------------------------------------------------------------- -if(MSVC) +if (MSVC) set_option_category("MSVC") define_option(MSVC_LINK_VERBOSE @@ -90,7 +90,7 @@ if(MSVC) OFF) define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF) -endif() +endif () #---------------------------------------------------------------------- @@ -99,9 +99,9 @@ set_option_category("Test and benchmark") unset(MILVUS_BUILD_TESTS CACHE) if (BUILD_UNIT_TEST) define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON) -else() +else () define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF) -endif(BUILD_UNIT_TEST) +endif (BUILD_UNIT_TEST) #---------------------------------------------------------------------- macro(config_summary) @@ -113,12 +113,12 @@ macro(config_summary) message(STATUS " Generator: ${CMAKE_GENERATOR}") message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") - if(${CMAKE_EXPORT_COMPILE_COMMANDS}) + if (${CMAKE_EXPORT_COMPILE_COMMANDS}) message( STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") - endif() + endif () - foreach(category ${MILVUS_OPTION_CATEGORIES}) + foreach (category ${MILVUS_OPTION_CATEGORIES}) message(STATUS) message(STATUS "${category} options:") @@ -126,50 +126,50 @@ macro(config_summary) set(option_names ${MILVUS_${category}_OPTION_NAMES}) set(max_value_length 0) - foreach(name ${option_names}) + foreach (name ${option_names}) string(LENGTH "\"${${name}}\"" value_length) - if(${max_value_length} LESS ${value_length}) + if (${max_value_length} LESS ${value_length}) set(max_value_length ${value_length}) - endif() - endforeach() + endif () + endforeach () - foreach(name ${option_names}) - if("${${name}_OPTION_TYPE}" STREQUAL "string") + foreach (name ${option_names}) + if ("${${name}_OPTION_TYPE}" STREQUAL "string") set(value "\"${${name}}\"") - else() + else () set(value "${${name}}") - endif() + endif () set(default ${${name}_OPTION_DEFAULT}) set(description ${${name}_OPTION_DESCRIPTION}) string(LENGTH ${description} description_length) - if(${description_length} LESS 70) + if (${description_length} LESS 70) string( SUBSTRING " " ${description_length} -1 description_padding) - else() + else () set(description_padding " ") - endif() + endif () set(comment "[${name}]") - if("${value}" STREQUAL "${default}") + if ("${value}" STREQUAL "${default}") set(comment "[default] ${comment}") - endif() + endif () - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set(comment "${comment} [${${name}_OPTION_ENUM}]") - endif() + endif () string( SUBSTRING "${value} " 0 ${max_value_length} value) message(STATUS " ${description} ${description_padding} ${value} ${comment}") - endforeach() + endforeach () - endforeach() + endforeach () endmacro() diff --git a/core/conf/server_config.template b/core/conf/server_config.template index bee0a67b27..8fc31366e3 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -32,9 +32,9 @@ cache_config: cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] cache_insert_data: false # whether to load inserted data into cache, must be a boolean -# Skip the following config if you are using GPU version - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] +#Uncomment the following config if you are using GPU version +# gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer +# gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] engine_config: use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times @@ -44,5 +44,4 @@ engine_config: resource_config: search_resources: # define the devices used for search computation, must be in format: cpu or gpux - cpu - - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu / gpux + index_build_device: cpu # CPU / GPU used for building index, must be in format: cpu / gpux diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 9e4065d646..79b5e0f1da 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -24,6 +24,9 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) add_subdirectory(index) +if (BUILD_FAISS_WITH_MKL) + add_compile_definitions("WITH_MKL") +endif () set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) foreach (dir ${INDEX_INCLUDE_DIRS}) diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index fcb291eda2..3f00e69fce 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -22,7 +22,7 @@ cmake_minimum_required(VERSION 3.14) message(STATUS "------------------------------KNOWHERE-----------------------------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") -set(KNOWHERE_VERSION "0.5.0") +set(KNOWHERE_VERSION "0.6.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}") project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) @@ -72,17 +72,17 @@ include(ExternalProject) include(DefineOptionsCore) include(BuildUtilsCore) -set(KNOWHERE_GPU_VERSION false) -if (MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION) - message(STATUS "Building Knowhere CPU version") - add_compile_definitions("MILVUS_CPU_VERSION") -else () +set(KNOWHERE_CPU_VERSION false) +if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION) message(STATUS "Building Knowhere GPU version") add_compile_definitions("MILVUS_GPU_VERSION") - set(KNOWHERE_GPU_VERSION true) enable_language(CUDA) find_package(CUDA 10 REQUIRED) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") +else () + message(STATUS "Building Knowhere CPU version") + set(KNOWHERE_CPU_VERSION true) + add_compile_definitions("MILVUS_CPU_VERSION") endif () include(ThirdPartyPackagesCore) diff --git a/core/src/index/cmake/DefineOptionsCore.cmake b/core/src/index/cmake/DefineOptionsCore.cmake index 99d1911d85..e49b3a779a 100644 --- a/core/src/index/cmake/DefineOptionsCore.cmake +++ b/core/src/index/cmake/DefineOptionsCore.cmake @@ -41,12 +41,12 @@ macro(define_option_string name description default) endmacro() #---------------------------------------------------------------------- -set_option_category("CPU version") +set_option_category("GPU version") -if (MILVUS_CPU_VERSION) - define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON) +if (MILVUS_GPU_VERSION) + define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON) else () - define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF) + define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF) endif () #---------------------------------------------------------------------- @@ -81,17 +81,6 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON) define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF) -#---------------------------------------------------------------------- -if (MSVC) - set_option_category("MSVC") - - define_option(MSVC_LINK_VERBOSE - "Pass verbose linking options when linking libraries and executables" - OFF) - - define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF) -endif () - #---------------------------------------------------------------------- set_option_category("Test and benchmark") diff --git a/core/src/main.cpp b/core/src/main.cpp index 9bb457cdda..c5b2d2dffe 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -51,7 +51,13 @@ print_banner() { std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; std::cout << std::endl; std::cout << "Welcome to Milvus!" << std::endl; - std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl; + std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with " +#ifdef WITH_MKL + << "MKL" +#else + << "OpenBLAS" +#endif + << " library." << std::endl; #ifdef MILVUS_CPU_VERSION std::cout << "You are using Milvus CPU version" << std::endl; #else @@ -137,7 +143,7 @@ main(int argc, char* argv[]) { s = server.Start(); if (s.ok()) { - std::cout << "Milvus server start successfully." << std::endl; + std::cout << "Milvus server started successfully!" << std::endl; } else { goto FAIL; } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 1d87e9aa6d..f241e661c8 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -25,6 +25,7 @@ #include "config/YamlConfigMgr.h" #include "server/Config.h" #include "utils/CommonUtil.h" +#include "utils/StringHelpFunctions.h" #include "utils/ValidationUtil.h" namespace milvus { @@ -343,6 +344,11 @@ Config::ResetDefaultConfig() { return s; } + s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } + s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); if (!s.ok()) { return s; @@ -403,8 +409,7 @@ Status Config::CheckServerConfigDeployMode(const std::string& value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, - "server_config.deploy_mode is not one of " - "single, cluster_readonly, and cluster_writable."); + "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."); } return Status::OK(); } @@ -592,15 +597,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int gpu_index; - Status s = GetResourceConfigIndexBuildDevice(gpu_index); + int device_id; + Status s = GetResourceConfigIndexBuildDevice(device_id); if (!s.ok()) { return s; } size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index); + if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); return Status(SERVER_UNEXPECTED_ERROR, msg); } else if (gpu_cache_capacity >= gpu_memory) { std::string msg = "Invalid gpu cache capacity: " + value + @@ -689,29 +694,33 @@ Config::CheckResourceConfigMode(const std::string& value) { } Status -CheckGpuDevice(const std::string& value) { +CheckResource(const std::string& value) { std::string s = value; std::transform(s.begin(), s.end(), s.begin(), ::tolower); + #ifdef MILVUS_CPU_VERSION if (s != "cpu") { return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s); } #else - const std::regex pat("gpu(\\d+)"); - std::cmatch m; - if (!std::regex_match(value.c_str(), m, pat)) { - std::string msg = "Invalid gpu device: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; + const std::regex pat("cpu|gpu(\\d+)"); + std::smatch m; + if (!std::regex_match(s, m, pat)) { + std::string msg = "Invalid search resource: " + value + + ". Possible reason: resource_config.search_resources is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t gpu_index = std::stoi(value.substr(3)); - if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid gpu device: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; - return Status(SERVER_INVALID_ARGUMENT, msg); + if (s.compare(0, 3, "gpu") == 0) { + int32_t gpu_index = std::stoi(s.substr(3)); + if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { + std::string msg = "Invalid search resource: " + value + + ". Possible reason: resource_config.search_resources does not match your hardware."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } } #endif + return Status::OK(); } @@ -724,38 +733,20 @@ Config::CheckResourceConfigSearchResources(const std::vector& value return Status(SERVER_INVALID_ARGUMENT, msg); } - bool cpu_found = false, gpu_found = false; - for (auto& device : value) { - if (device == "cpu") { - cpu_found = true; - continue; + for (auto& resource : value) { + auto status = CheckResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); } - if (CheckGpuDevice(device).ok()) { - gpu_found = true; - } else { - std::string msg = "Invalid search resource: " + device + - ". Possible reason: resource_config.search_resources does not match your hardware."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - - if (cpu_found && !gpu_found) { - std::string msg = - "Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource."; - return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - // if (value == "cpu") { - // return Status::OK(); - // } - if (!CheckGpuDevice(value).ok()) { - std::string msg = "Invalid index build device: " + value + - ". Possible reason: resource_config.index_build_device does not match your hardware."; - return Status(SERVER_INVALID_ARGUMENT, msg); + auto status = CheckResource(value); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); } return Status::OK(); } @@ -796,6 +787,22 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key return value; } +std::string +Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim, + const std::string& default_value) { + std::string value; + if (!GetConfigValueInMem(parent_key, child_key, value).ok()) { + std::vector sequence = GetConfigNode(parent_key).GetSequence(child_key); + if (sequence.empty()) { + value = default_value; + } else { + server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value); + } + SetConfigValueInMem(parent_key, child_key, value); + } + return value; +} + Status Config::GetServerConfigAddress(std::string& value) { value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); @@ -1019,8 +1026,10 @@ Config::GetResourceConfigMode(std::string& value) { Status Config::GetResourceConfigSearchResources(std::vector& value) { - ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); - value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES); + std::string str = + GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, + CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); return CheckResourceConfigSearchResources(value); } @@ -1033,10 +1042,10 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) { return s; } - if (str != "cpu") { - value = std::stoi(str.substr(3)); + if (str == "cpu") { + value = CPU_DEVICE_ID; } else { - value = -1; + value = std::stoi(str.substr(3)); } return Status::OK(); @@ -1163,7 +1172,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1174,7 +1183,7 @@ Config::SetMetricConfigCollector(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1185,7 +1194,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1197,7 +1206,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1208,7 +1217,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } @@ -1219,7 +1228,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1230,7 +1239,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); return Status::OK(); } @@ -1241,7 +1250,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1253,7 +1262,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1264,7 +1273,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1275,7 +1284,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } @@ -1287,7 +1296,21 @@ Config::SetResourceConfigMode(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + return Status::OK(); +} + +Status +Config::SetResourceConfigSearchResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); + + Status s = CheckResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } @@ -1298,7 +1321,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index a09939a9cb..b82614e0b9 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -92,12 +92,19 @@ static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; +#ifdef MILVUS_CPU_VERSION +static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; +#else +static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; +#endif static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; #ifdef MILVUS_CPU_VERSION static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; #else static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; #endif +const int32_t CPU_DEVICE_ID = -1; class Config { public: @@ -185,6 +192,9 @@ class Config { std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); + std::string + GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",", + const std::string& default_value = ""); public: /* server config */ @@ -306,6 +316,8 @@ class Config { Status SetResourceConfigMode(const std::string& value); Status + SetResourceConfigSearchResources(const std::string& value); + Status SetResourceConfigIndexBuildDevice(const std::string& value); private: diff --git a/core/src/utils/StringHelpFunctions.cpp b/core/src/utils/StringHelpFunctions.cpp index 230cc1a0ff..2db37829d6 100644 --- a/core/src/utils/StringHelpFunctions.cpp +++ b/core/src/utils/StringHelpFunctions.cpp @@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou } } -Status +void StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result) { if (str.empty()) { - return Status::OK(); + return; } - size_t last = 0; - size_t index = str.find_first_of(delimeter, last); - while (index != std::string::npos) { - result.emplace_back(str.substr(last, index - last)); - last = index + 1; - index = str.find_first_of(delimeter, last); + size_t prev = 0, pos = 0; + while (true) { + pos = str.find_first_of(delimeter, prev); + if (pos == std::string::npos) { + result.emplace_back(str.substr(prev)); + break; + } else { + result.emplace_back(str.substr(prev, pos - prev)); + prev = pos + 1; + } } - if (index - last > 0) { - std::string temp = str.substr(last); - result.emplace_back(temp); +} + +void +StringHelpFunctions::MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, + std::string& result) { + if (strs.empty()) { + result = ""; + return; } - return Status::OK(); + result = strs[0]; + for (size_t i = 1; i < strs.size(); i++) { + result = result + delimeter + strs[i]; + } } Status StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, std::vector& result) { if (quote.empty()) { - return SplitStringByDelimeter(str, delimeter, result); + SplitStringByDelimeter(str, delimeter, result); + return Status::OK(); } size_t last = 0; size_t index = str.find_first_of(quote, last); if (index == std::string::npos) { - return SplitStringByDelimeter(str, delimeter, result); + SplitStringByDelimeter(str, delimeter, result); + return Status::OK(); } std::string process_str = str; @@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin } if (!process_str.empty()) { - return SplitStringByDelimeter(process_str, delimeter, result); + SplitStringByDelimeter(process_str, delimeter, result); } return Status::OK(); diff --git a/core/src/utils/StringHelpFunctions.h b/core/src/utils/StringHelpFunctions.h index cb355332f1..0cb303bb4f 100644 --- a/core/src/utils/StringHelpFunctions.h +++ b/core/src/utils/StringHelpFunctions.h @@ -43,9 +43,12 @@ class StringHelpFunctions { // ,b, | b | // ,, | | // a a - static Status + static void SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); + static void + MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, std::string& result); + // assume the table has two columns, quote='\"', delimeter=',' // a,b a | b // "aa,gg,yy",b aa,gg,yy | b diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 7bcc21f7ee..e485bd729a 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -77,6 +77,7 @@ set(helper_files ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/Status.cpp + ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp ${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc ) diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index caaa66f979..5e6f61e543 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -22,6 +22,7 @@ #include "server/Config.h" #include "server/utils.h" #include "utils/CommonUtil.h" +#include "utils/StringHelpFunctions.h" #include "utils/ValidationUtil.h" namespace { @@ -98,6 +99,326 @@ TEST_F(ConfigTest, CONFIG_TEST) { ASSERT_TRUE(seqs.empty()); } +TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { + std::string config_path(CONFIG_PATH); + milvus::server::Config& config = milvus::server::Config::GetInstance(); + milvus::Status s; + std::string str_val; + int32_t int32_val; + int64_t int64_val; + float float_val; + bool bool_val; + + /* server config */ + std::string server_addr = "192.168.1.155"; + s = config.SetServerConfigAddress(server_addr); + ASSERT_TRUE(s.ok()); + s = config.GetServerConfigAddress(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == server_addr); + + std::string server_port = "12345"; + s = config.SetServerConfigPort(server_port); + ASSERT_TRUE(s.ok()); + s = config.GetServerConfigPort(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == server_port); + + std::string server_mode = "cluster_readonly"; + s = config.SetServerConfigDeployMode(server_mode); + ASSERT_TRUE(s.ok()); + s = config.GetServerConfigDeployMode(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == server_mode); + + std::string server_time_zone = "UTC+6"; + s = config.SetServerConfigTimeZone(server_time_zone); + ASSERT_TRUE(s.ok()); + s = config.GetServerConfigTimeZone(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == server_time_zone); + + /* db config */ + std::string db_primary_path = "/home/zilliz"; + s = config.SetDBConfigPrimaryPath(db_primary_path); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigPrimaryPath(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == db_primary_path); + + std::string db_secondary_path = "/home/zilliz"; + s = config.SetDBConfigSecondaryPath(db_secondary_path); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigSecondaryPath(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == db_secondary_path); + + std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus"; + s = config.SetDBConfigBackendUrl(db_backend_url); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigBackendUrl(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == db_backend_url); + + int32_t db_archive_disk_threshold = 100; + s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigArchiveDiskThreshold(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == db_archive_disk_threshold); + + int32_t db_archive_days_threshold = 365; + s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigArchiveDaysThreshold(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == db_archive_days_threshold); + + int32_t db_insert_buffer_size = 2; + s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); + ASSERT_TRUE(s.ok()); + s = config.GetDBConfigInsertBufferSize(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == db_insert_buffer_size); + + /* metric config */ + bool metric_enable_monitor = false; + s = config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor)); + ASSERT_TRUE(s.ok()); + s = config.GetMetricConfigEnableMonitor(bool_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(bool_val == metric_enable_monitor); + + std::string metric_collector = "prometheus"; + s = config.SetMetricConfigCollector(metric_collector); + ASSERT_TRUE(s.ok()); + s = config.GetMetricConfigCollector(str_val); + ASSERT_TRUE(str_val == metric_collector); + + std::string metric_prometheus_port = "2222"; + s = config.SetMetricConfigPrometheusPort(metric_prometheus_port); + ASSERT_TRUE(s.ok()); + s = config.GetMetricConfigPrometheusPort(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == metric_prometheus_port); + + /* cache config */ + int64_t cache_cpu_cache_capacity = 5; + s = config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity)); + ASSERT_TRUE(s.ok()); + s = config.GetCacheConfigCpuCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == cache_cpu_cache_capacity); + + float cache_cpu_cache_threshold = 0.1; + s = config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetCacheConfigCpuCacheThreshold(float_val); + ASSERT_TRUE(float_val == cache_cpu_cache_threshold); + + int64_t cache_gpu_cache_capacity = 1; + s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity)); + ASSERT_TRUE(s.ok()); + s = config.GetCacheConfigGpuCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == cache_gpu_cache_capacity); + + float cache_gpu_cache_threshold = 0.2; + s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetCacheConfigGpuCacheThreshold(float_val); + ASSERT_TRUE(float_val == cache_gpu_cache_threshold); + + bool cache_insert_data = true; + s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); + ASSERT_TRUE(s.ok()); + s = config.GetCacheConfigCacheInsertData(bool_val); + ASSERT_TRUE(bool_val == cache_insert_data); + + /* engine config */ + int32_t engine_use_blas_threshold = 50; + s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetEngineConfigUseBlasThreshold(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == engine_use_blas_threshold); + + int32_t engine_omp_thread_num = 8; + s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); + ASSERT_TRUE(s.ok()); + s = config.GetEngineConfigOmpThreadNum(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == engine_omp_thread_num); + + int32_t engine_gpu_search_threshold = 800; + s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetEngineConfigGpuSearchThreshold(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + + /* resource config */ + std::string resource_mode = "simple"; + s = config.SetResourceConfigMode(resource_mode); + ASSERT_TRUE(s.ok()); + s = config.GetResourceConfigMode(str_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(str_val == resource_mode); + +#ifdef MILVUS_CPU_VERSION + std::vector search_resources = {"cpu"}; +#else + std::vector search_resources = {"cpu", "gpu0"}; +#endif + std::vector res_vec; + std::string res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); + s = config.SetResourceConfigSearchResources(res_str); + ASSERT_TRUE(s.ok()); + s = config.GetResourceConfigSearchResources(res_vec); + ASSERT_TRUE(s.ok()); + for (size_t i = 0; i < search_resources.size(); i++) { + ASSERT_TRUE(search_resources[i] == res_vec[i]); + } + +#ifdef MILVUS_CPU_VERSION + int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; + s = config.SetResourceConfigIndexBuildDevice("cpu"); +#else + int32_t resource_index_build_device = 0; + s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); +#endif + ASSERT_TRUE(s.ok()); + s = config.GetResourceConfigIndexBuildDevice(int32_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int32_val == resource_index_build_device); +} + +TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { + std::string config_path(CONFIG_PATH); + milvus::server::Config& config = milvus::server::Config::GetInstance(); + milvus::Status s; + + s = config.LoadConfigFile(""); + ASSERT_FALSE(s.ok()); + + s = config.LoadConfigFile(config_path + INVALID_CONFIG_FILE); + ASSERT_FALSE(s.ok()); + s = config.LoadConfigFile(config_path + "dummy.yaml"); + ASSERT_FALSE(s.ok()); + + /* server config */ + s = config.SetServerConfigAddress("0.0.0"); + ASSERT_FALSE(s.ok()); + s = config.SetServerConfigAddress("0.0.0.256"); + ASSERT_FALSE(s.ok()); + + s = config.SetServerConfigPort("a"); + ASSERT_FALSE(s.ok()); + s = config.SetServerConfigPort("99999"); + ASSERT_FALSE(s.ok()); + + s = config.SetServerConfigDeployMode("cluster"); + ASSERT_FALSE(s.ok()); + + s = config.SetServerConfigTimeZone("GM"); + ASSERT_FALSE(s.ok()); + s = config.SetServerConfigTimeZone("GMT8"); + ASSERT_FALSE(s.ok()); + s = config.SetServerConfigTimeZone("UTCA"); + ASSERT_FALSE(s.ok()); + + /* db config */ + s = config.SetDBConfigPrimaryPath(""); + ASSERT_FALSE(s.ok()); + + // s = config.SetDBConfigSecondaryPath(""); + // ASSERT_FALSE(s.ok()); + + s = config.SetDBConfigBackendUrl("http://www.google.com"); + ASSERT_FALSE(s.ok()); + s = config.SetDBConfigBackendUrl("sqlite://:@:"); + ASSERT_FALSE(s.ok()); + s = config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus"); + ASSERT_FALSE(s.ok()); + + s = config.SetDBConfigArchiveDiskThreshold("0x10"); + ASSERT_FALSE(s.ok()); + + s = config.SetDBConfigArchiveDaysThreshold("0x10"); + ASSERT_FALSE(s.ok()); + + s = config.SetDBConfigInsertBufferSize("a"); + ASSERT_FALSE(s.ok()); + s = config.SetDBConfigInsertBufferSize("0"); + ASSERT_FALSE(s.ok()); + s = config.SetDBConfigInsertBufferSize("2048"); + ASSERT_FALSE(s.ok()); + + /* metric config */ + s = config.SetMetricConfigEnableMonitor("Y"); + ASSERT_FALSE(s.ok()); + + s = config.SetMetricConfigCollector("zilliz"); + ASSERT_FALSE(s.ok()); + + s = config.SetMetricConfigPrometheusPort("0xff"); + ASSERT_FALSE(s.ok()); + + /* cache config */ + s = config.SetCacheConfigCpuCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetCacheConfigCpuCacheCapacity("0"); + ASSERT_FALSE(s.ok()); + s = config.SetCacheConfigCpuCacheCapacity("2048"); + ASSERT_FALSE(s.ok()); + + s = config.SetCacheConfigCpuCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetCacheConfigCpuCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); + +#ifdef MILVUS_GPU_VERSION + s = config.SetCacheConfigGpuCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetCacheConfigGpuCacheCapacity("128"); + ASSERT_FALSE(s.ok()); + + s = config.SetCacheConfigGpuCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetCacheConfigGpuCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); +#endif + + s = config.SetCacheConfigCacheInsertData("N"); + ASSERT_FALSE(s.ok()); + + /* engine config */ + s = config.SetEngineConfigUseBlasThreshold("0xff"); + ASSERT_FALSE(s.ok()); + + s = config.SetEngineConfigOmpThreadNum("a"); + ASSERT_FALSE(s.ok()); + s = config.SetEngineConfigOmpThreadNum("10000"); + ASSERT_FALSE(s.ok()); + + s = config.SetEngineConfigGpuSearchThreshold("-1"); + ASSERT_FALSE(s.ok()); + + /* resource config */ + s = config.SetResourceConfigMode("default"); + ASSERT_FALSE(s.ok()); + + s = config.SetResourceConfigSearchResources("gpu10"); + ASSERT_FALSE(s.ok()); + + s = config.SetResourceConfigIndexBuildDevice("gup2"); + ASSERT_FALSE(s.ok()); + s = config.SetResourceConfigIndexBuildDevice("gpu16"); + ASSERT_FALSE(s.ok()); +} + TEST_F(ConfigTest, SERVER_CONFIG_TEST) { std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 36d0ab8597..2634a4af9c 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -117,12 +117,11 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { str = "a,b,c"; std::vector result; - auto status = milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); - ASSERT_TRUE(status.ok()); + milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); ASSERT_EQ(result.size(), 3UL); result.clear(); - status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); + auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); From 7fa712e45c4d5d5ed3db5ac2ae01696d9958c256 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 7 Nov 2019 16:11:55 +0800 Subject: [PATCH 2/2] Update DefineOptions.cmake --- core/cmake/DefineOptions.cmake | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index 4d5647b24d..6e05a12dd2 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -81,18 +81,6 @@ define_option(MILVUS_WITH_GRPC "Build with GRPC" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) -#---------------------------------------------------------------------- -if (MSVC) - set_option_category("MSVC") - - define_option(MSVC_LINK_VERBOSE - "Pass verbose linking options when linking libraries and executables" - OFF) - - define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF) -endif () - - #---------------------------------------------------------------------- set_option_category("Test and benchmark")