From f57033725b52d9f2c1f917b6edeabc0e893dd7e5 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 28 Jun 2019 10:52:16 +0800 Subject: [PATCH 1/4] change default db path Former-commit-id: e6337a98797a0b1361bc3e3949c56477a7693b42 --- cpp/.gitignore | 1 + cpp/CMakeLists.txt | 6 ++++++ cpp/conf/log_config.conf | 6 +++--- cpp/conf/log_config.template | 27 +++++++++++++++++++++++++++ cpp/conf/server_config.template | 28 ++++++++++++++++++++++++++++ cpp/conf/server_config.yaml | 4 ++-- 6 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 cpp/conf/log_config.template create mode 100644 cpp/conf/server_config.template diff --git a/cpp/.gitignore b/cpp/.gitignore index b9ba83ac8a..d8368bd79b 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,4 +1,5 @@ milvus/ conf/server_config.yaml +conf/log_config.conf version.h megasearch/ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 05cbc1db64..98c493b9a9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -141,6 +141,10 @@ endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) +set(MILVUS_PATH "/tmp/milvus") +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) + #install install(FILES scripts/start_server.sh @@ -153,4 +157,6 @@ install(FILES DESTINATION conf) + + config_summary() diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 80710b570e..29d46a7fe5 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -20,8 +20,8 @@ TO_STANDARD_OUTPUT = false ## Error logs * ERROR: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: - ENABLED = false - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template new file mode 100644 index 0000000000..ce421bac53 --- /dev/null +++ b/cpp/conf/log_config.template @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB +* DEBUG: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" +* TRACE: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template new file mode 100644 index 0000000000..b5f6ee2993 --- /dev/null +++ b/cpp/conf/server_config.template @@ -0,0 +1,28 @@ +server_config: + address: 0.0.0.0 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster + +db_config: + db_path: @MILVUS_PATH@ # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day + +metric_config: + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + + +license_config: # license configure + license_path: "@MILVUS_PATH@/system.license" # license file path + +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 1c26c0aa52..9019461940 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -22,7 +22,7 @@ metric_config: license_config: # license configure - license_path: "/tmp/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory From 61aedb86cc87e4931bf4f6dd57ddf2107bdb85d9 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 28 Jun 2019 11:06:27 +0800 Subject: [PATCH 2/4] change default db path Former-commit-id: ed8bbeb2ca73c5d2d4e4ec8b837cb948f18362fd --- cpp/CMakeLists.txt | 7 ++++++- cpp/build.sh | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 98c493b9a9..9a5c5b2f75 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -141,7 +141,12 @@ endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) -set(MILVUS_PATH "/tmp/milvus") + +if("${MILVUS_DB_PATHE}" STREQUAL "") + set(MILVUS_PATH "/tmp/milvus") +else() + set(MILVUS_PATH ${MILVUS_DB_PATHE}) +endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/cpp/build.sh b/cpp/build.sh index a7ece5fee6..de9a8d3205 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -6,6 +6,7 @@ LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" +DB_PATH="/opt/milvus" while getopts "p:t:uhlrc" arg do @@ -71,6 +72,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ + -DMILVUS_DB_PATHE=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} From 19409e8086db1eb6f456e098d630c8bc42b71499 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 28 Jun 2019 11:23:12 +0800 Subject: [PATCH 3/4] change default db path Former-commit-id: 7f345a1333859d9715fe31c69f80ae8bb67332d8 --- cpp/CMakeLists.txt | 8 +++----- cpp/build.sh | 24 ++++++++++++++---------- cpp/conf/log_config.template | 12 ++++++------ cpp/conf/server_config.template | 4 ++-- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9a5c5b2f75..f176ef7b27 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -135,17 +135,15 @@ if (BUILD_COVERAGE STREQUAL "ON") endif() -if (BUILD_UNIT_TEST) +if ("${BUILD_UNIT_TEST}" STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) -if("${MILVUS_DB_PATHE}" STREQUAL "") - set(MILVUS_PATH "/tmp/milvus") -else() - set(MILVUS_PATH ${MILVUS_DB_PATHE}) +if("${MILVUS_DB_PATH}" STREQUAL "") + set(MILVUS_DB_PATH "/tmp/milvus") endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/cpp/build.sh b/cpp/build.sh index de9a8d3205..80be1d7ddb 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -1,14 +1,14 @@ #!/bin/bash BUILD_TYPE="Debug" -BUILD_UNITTEST="off" +BUILD_UNITTEST="OFF" LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" -while getopts "p:t:uhlrc" arg +while getopts "p:d:t:uhlrc" arg do case $arg in t) @@ -16,11 +16,14 @@ do ;; u) echo "Build and run unittest cases" ; - BUILD_UNITTEST="on"; + BUILD_UNITTEST="ON"; ;; p) INSTALL_PREFIX=$OPTARG ;; + d) + DB_PATH=$OPTARG + ;; l) LICENSE_CHECK="ON" ;; @@ -37,12 +40,13 @@ do echo " parameter: --t: build type --u: building unit test options --p: install prefix --l: build license version --r: remove previous build directory --c: code coverage +-t: build type(default: Debug) +-u: building unit test options(default: OFF) +-p: install prefix(default: $(pwd)/milvus) +-d: db path(default: /opt/milvus) +-l: build license version(default: OFF) +-r: remove previous build directory(default: OFF) +-c: code coverage(default: OFF) usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] @@ -72,7 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ - -DMILVUS_DB_PATHE=${DB_PATH} \ + -DMILVUS_DB_PATH=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index ce421bac53..f4f3d3684c 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b5f6ee2993..1a1c8303f2 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -5,7 +5,7 @@ server_config: mode: single # milvus deployment type: single, cluster db_config: - db_path: @MILVUS_PATH@ # milvus data storage path + db_path: @MILVUS_DB_PATH@ # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB @@ -22,7 +22,7 @@ metric_config: license_config: # license configure - license_path: "@MILVUS_PATH@/system.license" # license file path + license_path: "@MILVUS_DB_PATH@/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file From c574b001b4eecc9feb47cfbb2814d348716e4d7a Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 28 Jun 2019 11:48:50 +0800 Subject: [PATCH 4/4] change default db path Former-commit-id: 7c96caed1379beadfad5fa7c144df05225176bd3 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6c33a9eb2f..8ef9aca97a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -24,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-122 - Archive criteria config - MS-124 - HasTable interface - MS-126 - Add more error code +- MS-128 - Change default db path ## New Feature